SherbyNet - /page/wikibook/entry.mu

<!DOCTYPE html>

<title>Batch normalization</title>

</head>

Batch normalization

</h1>

</header>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r1246091330">

/* start https://en.wikipedia.org/ */

.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r886047488">

/* start https://en.wikipedia.org/ */

.mw-parser-output .nobold{font-weight:normal}

/* end https://en.wikipedia.org/ */

</style><table class="sidebar sidebar-collapse nomobile nowraplinks"><tbody><tr><td class="sidebar-pretitle">Part of a series on</td></tr><tr><th class="sidebar-title-with-pretitle"><a href="Machine_learning" title="Machine learning">Machine learning</a> and <a href="Data_mining" title="Data mining">data mining</a></th></tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Paradigms</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Supervised_learning" title="Supervised learning">Supervised learning</a></li>

<li><a href="Unsupervised_learning" title="Unsupervised learning">Unsupervised learning</a></li>

<li><a href="Semi-supervised_learning" class="mw-redirect" title="Semi-supervised learning">Semi-supervised learning</a></li>

<li><a href="Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li>

<li><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></li>

<li><a href="Meta-learning_(computer_science)" title="Meta-learning (computer science)">Meta-learning</a></li>

<li><a href="Online_machine_learning" title="Online machine learning">Online learning</a></li>

<li><a href="Batch_learning" class="mw-redirect" title="Batch learning">Batch learning</a></li>

<li><a href="Curriculum_learning" title="Curriculum learning">Curriculum learning</a></li>

<li><a href="Rule-based_machine_learning" title="Rule-based machine learning">Rule-based learning</a></li>

<li><a href="Neuro-symbolic_AI" title="Neuro-symbolic AI">Neuro-symbolic AI</a></li>

<li><a href="Neuromorphic_engineering" class="mw-redirect" title="Neuromorphic engineering">Neuromorphic engineering</a></li>

<li><a href="Quantum_machine_learning" title="Quantum machine learning">Quantum machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Problems</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Statistical_classification" title="Statistical classification">Classification</a></li>

<li><a href="Generative_model" title="Generative model">Generative modeling</a></li>

<li><a href="Regression_analysis" title="Regression analysis">Regression</a></li>

<li><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></li>

<li><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></li>

<li><a href="Density_estimation" title="Density estimation">Density estimation</a></li>

<li><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></li>

<li><a href="Data_cleaning" class="mw-redirect" title="Data cleaning">Data cleaning</a></li>

<li><a href="Automated_machine_learning" title="Automated machine learning">AutoML</a></li>

<li><a href="Association_rule_learning" title="Association rule learning">Association rules</a></li>

<li><a href="Semantic_analysis_(machine_learning)" title="Semantic analysis (machine learning)">Semantic analysis</a></li>

<li><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></li>

<li><a href="Feature_engineering" title="Feature engineering">Feature engineering</a></li>

<li><a href="Feature_learning" title="Feature learning">Feature learning</a></li>

<li><a href="Learning_to_rank" title="Learning to rank">Learning to rank</a></li>

<li><a href="Grammar_induction" title="Grammar induction">Grammar induction</a></li>

<li><a href="Ontology_learning" title="Ontology learning">Ontology learning</a></li>

<li><a href="Multimodal_learning" title="Multimodal learning">Multimodal learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><div style="display: inline-block; line-height: 1.2em; padding: .1em 0;"><a href="Supervised_learning" title="Supervised learning">Supervised learning</a> (<a href="Statistical_classification" title="Statistical classification">classification</a> • <a href="Regression_analysis" title="Regression analysis">regression</a>) </div></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Apprenticeship_learning" title="Apprenticeship learning">Apprenticeship learning</a></li>

<li><a href="Decision_tree_learning" title="Decision tree learning">Decision trees</a></li>

<li><a href="Ensemble_learning" title="Ensemble learning">Ensembles</a>

<ul><li><a href="Bootstrap_aggregating" title="Bootstrap aggregating">Bagging</a></li>

<li><a href="Boosting_(machine_learning)" title="Boosting (machine learning)">Boosting</a></li>

<li><a href="Random_forest" title="Random forest">Random forest</a></li></ul></li>

<li><a href="Linear_regression" title="Linear regression">Linear regression</a></li>

<li><a href="Naive_Bayes_classifier" title="Naive Bayes classifier">Naive Bayes</a></li>

<li><a href="Artificial_neural_network" class="mw-redirect" title="Artificial neural network">Artificial neural networks</a></li>

<li><a href="Logistic_regression" title="Logistic regression">Logistic regression</a></li>

<li><a href="Perceptron" title="Perceptron">Perceptron</a></li>

<li><a href="Relevance_vector_machine" title="Relevance vector machine">Relevance vector machine (RVM)</a></li>

<li><a href="Support_vector_machine" title="Support vector machine">Support vector machine (SVM)</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="BIRCH" title="BIRCH">BIRCH</a></li>

<li><a href="Hierarchical_clustering" title="Hierarchical clustering">Hierarchical</a></li>

<li><a href="K-means_clustering" title="K-means clustering">k-means</a></li>

<li><a href="Fuzzy_clustering" title="Fuzzy clustering">Fuzzy</a></li>

<li><a href="Expectation%E2%80%93maximization_algorithm" title="Expectation–maximization algorithm">Expectation–maximization (EM)</a></li>

<li> <a href="DBSCAN" title="DBSCAN">DBSCAN</a></li>

<li><a href="OPTICS_algorithm" title="OPTICS algorithm">OPTICS</a></li>

<li><a href="Mean_shift" title="Mean shift">Mean shift</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Factor_analysis" title="Factor analysis">Factor analysis</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Graphical_model" title="Graphical model">Graphical models</a>

<ul><li><a href="Bayesian_network" title="Bayesian network">Bayes net</a></li>

<li><a href="Conditional_random_field" title="Conditional random field">Conditional random field</a></li>

<li><a href="Hidden_Markov_model" title="Hidden Markov model">Hidden Markov</a></li></ul></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Random_sample_consensus" title="Random sample consensus">RANSAC</a></li>

<li><a href="Local_outlier_factor" title="Local outlier factor">Local outlier factor</a></li>

<li><a href="Isolation_forest" title="Isolation forest">Isolation forest</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">Neural networks</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Deep_learning" title="Deep learning">Deep learning</a></li>

<li><a href="Feedforward_neural_network" title="Feedforward neural network">Feedforward neural network</a></li>

<li><a href="Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network</a>

<li><a href="Reservoir_computing" title="Reservoir computing">reservoir computing</a></li></ul></li>

<li><a href="Boltzmann_machine" title="Boltzmann machine">Boltzmann machine</a>

<ul><li><a href="Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">Restricted</a></li></ul></li>

<li><a href="Diffusion_model" title="Diffusion model">Diffusion model</a></li>

<li><a href="Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network</a>

<li><a href="LeNet" title="LeNet">LeNet</a></li>

<li><a href="AlexNet" title="AlexNet">AlexNet</a></li>

<li><a href="DeepDream" title="DeepDream">DeepDream</a></li></ul></li>

<li><a href="Neural_field" title="Neural field">Neural field</a>

<ul><li><a href="Neural_radiance_field" title="Neural radiance field">Neural radiance field</a></li>

<li><a href="Physics-informed_neural_networks" title="Physics-informed neural networks">Physics-informed neural networks</a></li></ul></li>

<li><a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>

<ul><li><a href="Vision_transformer" title="Vision transformer">Vision</a></li></ul></li>

<li><a href="Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li>

<li><a href="Spiking_neural_network" title="Spiking neural network">Spiking neural network</a></li>

<li><a href="Memtransistor" title="Memtransistor">Memtransistor</a></li>

<li><a href="Electrochemical_RAM" title="Electrochemical RAM">Electrochemical RAM</a> (ECRAM)</li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Q-learning" title="Q-learning">Q-learning</a></li>

<li><a href="Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li>

<li><a href="State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li>

<li><a href="Temporal_difference_learning" title="Temporal difference learning">Temporal difference (TD)</a></li>

<li><a href="Multi-agent_reinforcement_learning" title="Multi-agent reinforcement learning">Multi-agent</a>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Learning with humans</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Active_learning_(machine_learning)" title="Active learning (machine learning)">Active learning</a></li>

<li><a href="Crowdsourcing" title="Crowdsourcing">Crowdsourcing</a></li>

<li><a href="Human-in-the-loop" title="Human-in-the-loop">Human-in-the-loop</a></li>

<li><a href="Mechanistic_interpretability" title="Mechanistic interpretability">Mechanistic interpretability</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Model diagnostics</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Coefficient_of_determination" title="Coefficient of determination">Coefficient of determination</a></li>

<li><a href="Confusion_matrix" title="Confusion matrix">Confusion matrix</a></li>

<li><a href="Learning_curve_(machine_learning)" title="Learning curve (machine learning)">Learning curve</a></li>

<li><a href="Receiver_operating_characteristic" title="Receiver operating characteristic">ROC curve</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Mathematical foundations</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Kernel_machines" class="mw-redirect" title="Kernel machines">Kernel machines</a></li>

<li><a href="Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li>

<li><a href="Computational_learning_theory" title="Computational learning theory">Computational learning theory</a></li>

<li><a href="Empirical_risk_minimization" title="Empirical risk minimization">Empirical risk minimization</a></li>

<li><a href="Occam_learning" title="Occam learning">Occam learning</a></li>

<li><a href="Probably_approximately_correct_learning" title="Probably approximately correct learning">PAC learning</a></li>

<li><a href="Statistical_learning_theory" title="Statistical learning theory">Statistical learning</a></li>

<li><a href="Vapnik%E2%80%93Chervonenkis_theory" title="Vapnik–Chervonenkis theory">VC theory</a></li>

<li><a href="Topological_deep_learning" title="Topological deep learning">Topological deep learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Journals and conferences</div><div class="sidebar-list-content mw-collapsible-content hlist">

<li><a href="Conference_on_Neural_Information_Processing_Systems" title="Conference on Neural Information Processing Systems">NeurIPS</a></li>

<li><a href="International_Joint_Conference_on_Artificial_Intelligence" title="International Joint Conference on Artificial Intelligence">IJCAI</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Related articles</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Glossary_of_artificial_intelligence" title="Glossary of artificial intelligence">Glossary of artificial intelligence</a></li>

<li><a href="List_of_datasets_for_machine-learning_research" title="List of datasets for machine-learning research">List of datasets for machine-learning research</a>

<ul><li><a href="List_of_datasets_in_computer_vision_and_image_processing" title="List of datasets in computer vision and image processing">List of datasets in computer vision and image processing</a></li></ul></li>

<li><a href="Outline_of_machine_learning" title="Outline of machine learning">Outline of machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-navbar"><style data-mw-deduplicate="TemplateStyles:r1239400231">

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></td></tr></tbody></table>

Batch normalization (also known as batch norm) is a <a href="Normalization_(machine_learning)" title="Normalization (machine learning)">normalization</a> technique used to make training of <a href="Artificial_neural_network" class="mw-redirect" title="Artificial neural network">artificial neural networks</a> faster and more stable by adjusting the inputs to each layer—re-centering them around zero and re-scaling them to a standard size. It was introduced by Sergey Ioffe and Christian Szegedy in 2015.<a href="#cite_note-:0-1">[1]</a>

Experts still debate why batch normalization works so well. It was initially thought to tackle internal covariate shift, a problem where parameter initialization and changes in the distribution of the inputs of each layer affect the learning rate of the network.<a href="#cite_note-:0-1">[1]</a> However, newer research suggests it doesn’t fix this shift but instead smooths the <a href="Intelligent_agent#Objective_function" title="Intelligent agent">objective function</a>—a mathematical guide the network follows to improve—enhancing performance.<a href="#cite_note-:13-2">[2]</a> In very deep networks, batch normalization can initially cause a severe <a href="Vanishing_gradient_problem" title="Vanishing gradient problem">gradient explosion</a>—where updates to the network grow uncontrollably large—but this is managed with shortcuts called skip connections in residual networks.<a href="#cite_note-:73-3">[3]</a> Another theory is that batch normalization adjusts data by handling its size and path separately, speeding up training.<a href="#cite_note-:23-4">[4]</a>

<div class="mw-heading mw-heading2"><h2 id="Internal_covariate_shift">Internal covariate shift</h2></div>

Each layer in a neural network has inputs that follow a specific distribution, which shifts during training due to two main factors: the random starting values of the network’s settings (<a href="Parameter_initialization" class="mw-redirect" title="Parameter initialization">parameter initialization</a>) and the natural variation in the input data. This shifting pattern affecting the inputs to the network’s inner layers is called internal covariate shift. While a strict definition isn’t fully agreed upon, experiments show that it involves changes in the means and variances of these inputs during training.

Batch normalization was first developed to address internal covariate shift.<a href="#cite_note-:0-1">[1]</a> During training, as the parameters of preceding layers adjust, the distribution of inputs to the current layer changes accordingly, such that the current layer needs to constantly readjust to new distributions. This issue is particularly severe in deep networks, because small changes in shallower hidden layers will be amplified as they propagate within the network, resulting in significant shift in deeper hidden layers. Batch normalization was proposed to reduced these unwanted shifts to speed up training and produce more reliable models.

Beyond possibly tackling internal covariate shift, batch normalization offers several additional advantages. It allows the network to use a higher <a href="Learning_rate" title="Learning rate">learning rate</a>—a setting that controls how quickly the network learns—without causing problems like vanishing or exploding gradients, where updates become too small or too large. It also appears to have a regularizing effect, improving the network’s ability to generalize to new data, reducing the need for <a href="Dropout_(neural_networks)" class="mw-redirect" title="Dropout (neural networks)">dropout</a>, a technique used to prevent <a href="Overfitting" title="Overfitting">overfitting</a> (when a model learns the training data too well and fails on new data). Additionally, networks using batch normalization are less sensitive to the choice of starting settings or learning rates, making them more robust and adaptable.

<div class="mw-heading mw-heading2"><h2 id="Procedures">Procedures</h2></div>

<div class="mw-heading mw-heading3"><h3 id="Transformation">Transformation</h3></div>

In a neural network, batch normalization is achieved through a normalization step that fixes the means and variances of each layer's inputs. Ideally, the normalization would be conducted over the entire training set, but to use this step jointly with <a href="Stochastic_optimization" title="Stochastic optimization">stochastic optimization</a> methods, it is impractical to use the global information. Thus, normalization is restrained to each mini-batch in the training process.

Let us use B to denote a mini-batch of size m of the entire training set. The empirical <a href="Mean" title="Mean">mean</a> and <a href="Variance" title="Variance">variance</a> of B could thus be denoted as

<msub>

</mrow>

</msub>

<mfrac>

</mfrac>

</mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mu _{B}={\frac {1}{m}}\sum _{i=1}^{m}x_{i}}</annotation>

</semantics>

</math><img src="./cd0d3984435337cf2f54a8a37a8b59a2c9be3962.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:15.115ex; height:6.843ex;" alt="{\displaystyle \mu _{B}={\frac {1}{m}}\sum _{i=1}^{m}x_{i}}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma _{B}^{2}={\frac {1}{m}}\sum _{i=1}^{m}(x_{i}-\mu _{B})^{2}}">

</mrow>

</mrow>

</msubsup>

<mfrac>

</mfrac>

</mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma _{B}^{2}={\frac {1}{m}}\sum _{i=1}^{m}(x_{i}-\mu _{B})^{2}}</annotation>

</semantics>

</math><img src="./ee0c29a63307bef096890d6afa3022f9a1ceb1b1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:23.239ex; height:6.843ex;" alt="{\displaystyle \sigma _{B}^{2}={\frac {1}{m}}\sum _{i=1}^{m}(x_{i}-\mu _{B})^{2}}" loading="lazy">.

For a layer of the network with d-dimensional input, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x=(x^{(1)},...,x^{(d)})}">

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle x=(x^{(1)},...,x^{(d)})}</annotation>

</semantics>

</math><img src="./78d919bcd4cf8569fc248bad8506dc37c932ad80.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:18.771ex; height:3.343ex;" alt="{\displaystyle x=(x^{(1)},...,x^{(d)})}" loading="lazy">, each dimension of its input is then normalized (i.e. re-centered and re-scaled) separately,

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</msubsup>

<mfrac>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

<msqrt>

<msup>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msup>

</msqrt>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {x}}_{i}^{(k)}={\frac {x_{i}^{(k)}-\mu _{B}^{(k)}}{\sqrt {\left(\sigma _{B}^{(k)}\right)^{2}+\epsilon }}}}</annotation>

</semantics>

</math><img src="./5bcadd141b0b7eab920a11a6f3a773493d35d744.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -6.005ex; width:21.269ex; height:10.676ex;" alt="{\displaystyle {\hat {x}}_{i}^{(k)}={\frac {x_{i}^{(k)}-\mu _{B}^{(k)}}{\sqrt {\left(\sigma _{B}^{(k)}\right)^{2}+\epsilon }}}}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle k\in [1,d]}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle k\in [1,d]}</annotation>

</semantics>

</math><img src="./198a4d2d8ce137b1f4242c3f5cce495d77d7b0fd.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.758ex; height:2.843ex;" alt="{\displaystyle k\in [1,d]}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i\in [1,m]}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i\in [1,m]}</annotation>

</semantics>

</math><img src="./ab916af4b9efb98e62b719bad44c18601285097b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.174ex; height:2.843ex;" alt="{\displaystyle i\in [1,m]}" loading="lazy">; <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mu _{B}^{(k)}}">

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mu _{B}^{(k)}}</annotation>

</semantics>

</math><img src="./bc97d7418b5f8c2b6396dd94f491ed9a21d8ecb0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.77ex; height:3.676ex;" alt="{\displaystyle \mu _{B}^{(k)}}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma _{B}^{(k)}}">

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma _{B}^{(k)}}</annotation>

</semantics>

</math><img src="./977743d2c31d299bbf32990e40a8410bb12b16dd.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.699ex; height:3.676ex;" alt="{\displaystyle \sigma _{B}^{(k)}}" loading="lazy"> are the per-dimension mean and standard deviation, respectively.

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \epsilon }</annotation>

</semantics>

</math><img src="./c3837cad72483d97bcdde49c85d3b7b859fb3fd2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.944ex; height:1.676ex;" alt="{\displaystyle \epsilon }" loading="lazy"> is added in the denominator for numerical stability and is an arbitrarily small constant. The resulting normalized activation <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {x}}^{(k)}}">

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {x}}^{(k)}}</annotation>

</semantics>

</math><img src="./e006f6308fa590163ebd3e53744a83575ca56450.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.698ex; height:2.843ex;" alt="{\displaystyle {\hat {x}}^{(k)}}" loading="lazy"> have zero mean and unit variance, if <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \epsilon }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \epsilon }</annotation>

</semantics>

</math><img src="./c3837cad72483d97bcdde49c85d3b7b859fb3fd2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.944ex; height:1.676ex;" alt="{\displaystyle \epsilon }" loading="lazy"> is not taken into account. To restore the representation power of the network, a transformation step then follows as

</mrow>

</mrow>

</msubsup>

<msup>

</mrow>

</msup>

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</msubsup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y_{i}^{(k)}=\gamma ^{(k)}{\hat {x}}_{i}^{(k)}+\beta ^{(k)}}</annotation>

</semantics>

</math><img src="./e890712d3a79c32441a69dba2188cbb4efc41bec.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:20.518ex; height:3.676ex;" alt="{\displaystyle y_{i}^{(k)}=\gamma ^{(k)}{\hat {x}}_{i}^{(k)}+\beta ^{(k)}}" loading="lazy">,

where the parameters <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma ^{(k)}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma ^{(k)}}</annotation>

</semantics>

</math><img src="./c4592b76ab91a30f10dec9195b6544c05caf8453.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:3.648ex; height:3.343ex;" alt="{\displaystyle \gamma ^{(k)}}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta ^{(k)}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta ^{(k)}}</annotation>

</semantics>

</math><img src="./c335496d9ec4863fc99f5ac2c836d26e7526ccd7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.705ex; height:3.176ex;" alt="{\displaystyle \beta ^{(k)}}" loading="lazy"> are subsequently learned in the optimization process.

Formally, the operation that implements batch normalization is a transform <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle BN_{\gamma ^{(k)},\beta ^{(k)}}:x_{1...m}^{(k)}\rightarrow y_{1...m}^{(k)}}">

<msub>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mrow>

</msub>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle BN_{\gamma ^{(k)},\beta ^{(k)}}:x_{1...m}^{(k)}\rightarrow y_{1...m}^{(k)}}</annotation>

</semantics>

</math><img src="./e9484456b00b1986f422a5e7bba1bdac2df29b56.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.505ex; width:25.725ex; height:4.176ex;" alt="{\displaystyle BN_{\gamma ^{(k)},\beta ^{(k)}}:x_{1...m}^{(k)}\rightarrow y_{1...m}^{(k)}}" loading="lazy"> called the Batch Normalizing transform. The output of the BN transform <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y^{(k)}=BN_{\gamma ^{(k)},\beta ^{(k)}}(x^{(k)})}">

<msup>

</mrow>

</msup>

<msub>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y^{(k)}=BN_{\gamma ^{(k)},\beta ^{(k)}}(x^{(k)})}</annotation>

</semantics>

</math><img src="./aa553bd532d824a0963c3a0e79e7e2444f0dd87a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.505ex; width:22.101ex; height:4.009ex;" alt="{\displaystyle y^{(k)}=BN_{\gamma ^{(k)},\beta ^{(k)}}(x^{(k)})}" loading="lazy"> is then passed to other network layers, while the normalized output <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {x}}_{i}^{(k)}}">

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {x}}_{i}^{(k)}}</annotation>

</semantics>

</math><img src="./8d90e66232261cb252f1eb084812a93691750eff.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.698ex; height:3.676ex;" alt="{\displaystyle {\hat {x}}_{i}^{(k)}}" loading="lazy"> remains internal to the current layer.

<div class="mw-heading mw-heading3"><h3 id="Backpropagation">Backpropagation</h3></div>

The described BN transform is a <a href="Differentiable_function" title="Differentiable function">differentiable</a> operation, and the gradient of the <a href="Loss_function" title="Loss function">loss</a> l with respect to the different parameters can be computed directly with the <a href="Chain_rule" title="Chain rule">chain rule</a>.

Specifically, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial l}{\partial y_{i}^{(k)}}}}">

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial l}{\partial y_{i}^{(k)}}}}</annotation>

</semantics>

</math><img src="./a3724b90285aa0b2efee4825194ff04d39a5fd29.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:5.683ex; height:7.009ex;" alt="{\displaystyle {\frac {\partial l}{\partial y_{i}^{(k)}}}}" loading="lazy"> depends on the choice of <a href="Activation_function" title="Activation function">activation function</a>, and the <a href="Gradient" title="Gradient">gradient</a> against other parameters could be expressed as a function of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial l}{\partial y_{i}^{(k)}}}}">

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial l}{\partial y_{i}^{(k)}}}}</annotation>

</semantics>

</math><img src="./a3724b90285aa0b2efee4825194ff04d39a5fd29.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:5.683ex; height:7.009ex;" alt="{\displaystyle {\frac {\partial l}{\partial y_{i}^{(k)}}}}" loading="lazy">:

<mfrac>

<mrow>

</mrow>

<mrow>

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial l}{\partial {\hat {x}}_{i}^{(k)}}}={\frac {\partial l}{\partial y_{i}^{(k)}}}\gamma ^{(k)}}</annotation>

</semantics>

</math><img src="./32b1beb5cc60115e019d4d9538b9d566ac00dcb4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:18.281ex; height:7.009ex;" alt="{\displaystyle {\frac {\partial l}{\partial {\hat {x}}_{i}^{(k)}}}={\frac {\partial l}{\partial y_{i}^{(k)}}}\gamma ^{(k)}}" loading="lazy">,

<mfrac>

<mrow>

</mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

</munderover>

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial l}{\partial \gamma ^{(k)}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}{\hat {x}}_{i}^{(k)}}</annotation>

</semantics>

</math><img src="./58b561531b36aa285816ce40e650f5a434128f8b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:22.023ex; height:7.343ex;" alt="{\displaystyle {\frac {\partial l}{\partial \gamma ^{(k)}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}{\hat {x}}_{i}^{(k)}}" loading="lazy">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial l}{\partial \beta ^{(k)}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}}">

<mfrac>

<mrow>

</mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

</munderover>

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial l}{\partial \beta ^{(k)}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}}</annotation>

</semantics>

</math><img src="./33a38ea1fbf5cb13f44c41fd90c1e056e400cc7e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:18.383ex; height:7.343ex;" alt="{\displaystyle {\frac {\partial l}{\partial \beta ^{(k)}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}}" loading="lazy">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial l}{\partial \sigma _{B}^{(k)^{2}}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}(x_{i}^{(k)}-\mu _{B}^{(k)})\left(-{\frac {\gamma ^{(k)}}{2}}(\sigma _{B}^{(k)^{2}}+\epsilon )^{-3/2}\right)}">

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

</munderover>

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

<mrow>

<mrow>

<mfrac>

<msup>

</mrow>

</msup>

</mfrac>

</mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msubsup>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial l}{\partial \sigma _{B}^{(k)^{2}}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}(x_{i}^{(k)}-\mu _{B}^{(k)})\left(-{\frac {\gamma ^{(k)}}{2}}(\sigma _{B}^{(k)^{2}}+\epsilon )^{-3/2}\right)}</annotation>

</semantics>

</math><img src="./d7c327087d5fd83d97e4aafb42fbafb069abac12.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.838ex; width:55.786ex; height:8.176ex;" alt="{\displaystyle {\frac {\partial l}{\partial \sigma _{B}^{(k)^{2}}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}(x_{i}^{(k)}-\mu _{B}^{(k)})\left(-{\frac {\gamma ^{(k)}}{2}}(\sigma _{B}^{(k)^{2}}+\epsilon )^{-3/2}\right)}" loading="lazy">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial l}{\partial \mu _{B}^{(k)}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}{\frac {-\gamma ^{(k)}}{\sqrt {\sigma _{B}^{(k)^{2}}+\epsilon }}}+{\frac {\partial l}{\partial \sigma _{B}^{(k)^{2}}}}{\frac {1}{m}}\sum _{i=1}^{m}(-2)\cdot (x_{i}^{(k)}-\mu _{B}^{(k)})}">

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

</munderover>

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<mfrac>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

<msqrt>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msubsup>

</msqrt>

</mfrac>

</mrow>

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<mfrac>

</mfrac>

</mrow>

</mrow>

</mrow>

</munderover>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial l}{\partial \mu _{B}^{(k)}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}{\frac {-\gamma ^{(k)}}{\sqrt {\sigma _{B}^{(k)^{2}}+\epsilon }}}+{\frac {\partial l}{\partial \sigma _{B}^{(k)^{2}}}}{\frac {1}{m}}\sum _{i=1}^{m}(-2)\cdot (x_{i}^{(k)}-\mu _{B}^{(k)})}</annotation>

</semantics>

</math><img src="./b6a39565a8ef5c686e1525bfd66f02c4f48d7b23.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -4.671ex; width:64.641ex; height:8.676ex;" alt="{\displaystyle {\frac {\partial l}{\partial \mu _{B}^{(k)}}}=\sum _{i=1}^{m}{\frac {\partial l}{\partial y_{i}^{(k)}}}{\frac {-\gamma ^{(k)}}{\sqrt {\sigma _{B}^{(k)^{2}}+\epsilon }}}+{\frac {\partial l}{\partial \sigma _{B}^{(k)^{2}}}}{\frac {1}{m}}\sum _{i=1}^{m}(-2)\cdot (x_{i}^{(k)}-\mu _{B}^{(k)})}" loading="lazy">,

and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial l}{\partial x_{i}^{(k)}}}={\frac {\partial l}{\partial {\hat {x}}_{i}^{(k)}}}{\frac {1}{\sqrt {\sigma _{B}^{(k)^{2}}+\epsilon }}}+{\frac {\partial l}{\partial \sigma _{B}^{(k)^{2}}}}{\frac {2(x_{i}^{(k)}-\mu _{B}^{(k)})}{m}}+{\frac {\partial l}{\partial \mu _{B}^{(k)}}}{\frac {1}{m}}}">

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<mfrac>

<mrow>

</mrow>

<mrow>

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<mfrac>

<msqrt>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msubsup>

</msqrt>

</mfrac>

</mrow>

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<mfrac>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<mfrac>

<mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mfrac>

</mrow>

<mfrac>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial l}{\partial x_{i}^{(k)}}}={\frac {\partial l}{\partial {\hat {x}}_{i}^{(k)}}}{\frac {1}{\sqrt {\sigma _{B}^{(k)^{2}}+\epsilon }}}+{\frac {\partial l}{\partial \sigma _{B}^{(k)^{2}}}}{\frac {2(x_{i}^{(k)}-\mu _{B}^{(k)})}{m}}+{\frac {\partial l}{\partial \mu _{B}^{(k)}}}{\frac {1}{m}}}</annotation>

</semantics>

</math><img src="./d95ac93be5adf1d3ba002490414507748ad61fd5.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -4.671ex; width:61.558ex; height:9.343ex;" alt="{\displaystyle {\frac {\partial l}{\partial x_{i}^{(k)}}}={\frac {\partial l}{\partial {\hat {x}}_{i}^{(k)}}}{\frac {1}{\sqrt {\sigma _{B}^{(k)^{2}}+\epsilon }}}+{\frac {\partial l}{\partial \sigma _{B}^{(k)^{2}}}}{\frac {2(x_{i}^{(k)}-\mu _{B}^{(k)})}{m}}+{\frac {\partial l}{\partial \mu _{B}^{(k)}}}{\frac {1}{m}}}" loading="lazy">.

<div class="mw-heading mw-heading3"><h3 id="Inference">Inference</h3></div>

During the training stage, the normalization steps depend on the mini-batches to ensure efficient and reliable training. However, in the inference stage, this dependence is not useful any more. Instead, the normalization step in this stage is computed with the population statistics such that the output could depend on the input in a deterministic manner. The population mean, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle E[x^{(k)}]}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle E[x^{(k)}]}</annotation>

</semantics>

</math><img src="./9de884a23a219434520a8f5d2fa6029aa08d9229.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.767ex; height:3.343ex;" alt="{\displaystyle E[x^{(k)}]}" loading="lazy">, and variance, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {Var} [x^{(k)}]}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {Var} [x^{(k)}]}</annotation>

</semantics>

</math><img src="./9626fe6237e75a52277fdfc8f0bde740b22106b2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.809ex; height:3.343ex;" alt="{\displaystyle \operatorname {Var} [x^{(k)}]}" loading="lazy">, are computed as:

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle E[x^{(k)}]=E_{B}[\mu _{B}^{(k)}]}</annotation>

</semantics>

</math><img src="./924410b15e63683d9d19aeccadb2d57ca423f151.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:18.124ex; height:3.676ex;" alt="{\displaystyle E[x^{(k)}]=E_{B}[\mu _{B}^{(k)}]}" loading="lazy">, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {Var} [x^{(k)}]={\frac {m}{m-1}}E_{B}[\left(\sigma _{B}^{(k)}\right)^{2}]}">

<msup>

</mrow>

</msup>

<mfrac>

<mrow>

</mrow>

</mfrac>

</mrow>

<msub>

</mrow>

</msub>

<msup>

<mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {Var} [x^{(k)}]={\frac {m}{m-1}}E_{B}[\left(\sigma _{B}^{(k)}\right)^{2}]}</annotation>

</semantics>

</math><img src="./1717aca2b768b32fe46c3c8f9c66daa25f953dff.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.005ex; width:30.803ex; height:5.343ex;" alt="{\displaystyle \operatorname {Var} [x^{(k)}]={\frac {m}{m-1}}E_{B}[\left(\sigma _{B}^{(k)}\right)^{2}]}" loading="lazy">.

The population statistics thus is a complete representation of the mini-batches.

The BN transform in the inference step thus becomes

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</msubsup>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

<mfrac>

<mrow>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mrow>

<msqrt>

<msup>

</mrow>

</msup>

</msqrt>

</mfrac>

</mrow>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y^{(k)}=BN_{\gamma ^{(k)},\beta ^{(k)}}^{\text{inf}}(x^{(k)})=\gamma ^{(k)}{\frac {x^{(k)}-E[x^{(k)}]}{\sqrt {\operatorname {Var} [x^{(k)}]+\epsilon }}}+\beta ^{(k)}}</annotation>

</semantics>

</math><img src="./032e23f561aa029ae0936e1b55f38a33f4488dcd.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -4.671ex; width:51.146ex; height:8.843ex;" alt="{\displaystyle y^{(k)}=BN_{\gamma ^{(k)},\beta ^{(k)}}^{\text{inf}}(x^{(k)})=\gamma ^{(k)}{\frac {x^{(k)}-E[x^{(k)}]}{\sqrt {\operatorname {Var} [x^{(k)}]+\epsilon }}}+\beta ^{(k)}}" loading="lazy">,

where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y^{(k)}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y^{(k)}}</annotation>

</semantics>

</math><img src="./6187d5d10a29c1047e991615c8aaa040219f4741.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.529ex; height:3.176ex;" alt="{\displaystyle y^{(k)}}" loading="lazy"> is passed on to future layers instead of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x^{(k)}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle x^{(k)}}</annotation>

</semantics>

</math><img src="./4c4ad77f04c0f250f01c3b6b5195ac5769d9dacf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.698ex; height:2.843ex;" alt="{\displaystyle x^{(k)}}" loading="lazy">. Since the parameters are fixed in this transformation, the batch normalization procedure is essentially applying a <a href="Linear_map" title="Linear map">linear transform</a> to the activation.

<div class="mw-heading mw-heading2"><h2 id="Theory">Theory</h2></div>

Although batch normalization has become popular due to its strong empirical performance, the working mechanism of the method is not yet well-understood. The explanation made in the original paper<a href="#cite_note-:0-1">[1]</a> was that batch norm works by reducing internal covariate shift, but this has been challenged by more recent work. One experiment<a href="#cite_note-:1-5">[5]</a> trained a VGG-16 network<a href="#cite_note-VGG-6">[6]</a> under 3 different training regimes: standard (no batch norm), batch norm, and batch norm with noise added to each layer during training. In the third model, the noise has non-zero mean and non-unit variance, i.e. it explicitly introduces covariate shift. Despite this, it showed similar accuracy to the second model, and both performed better than the first, suggesting that covariate shift is not the reason that batch norm improves performance.

Using batch normalization causes the items in a batch to no longer be <a href="Independent_and_identically_distributed_random_variables" title="Independent and identically distributed random variables">iid</a>, which can lead to difficulties in training due to lower quality gradient estimation.<a href="#cite_note-7">[7]</a>

<div class="mw-heading mw-heading3"><h3 id="Smoothness">Smoothness</h3></div>

One alternative explanation,<a href="#cite_note-:1-5">[5]</a> is that the improvement with batch normalization is instead due to it producing a smoother parameter space and smoother gradients, as formalized by a smaller <a href="Lipschitz_continuity" title="Lipschitz continuity">Lipschitz constant</a>.

Consider two identical networks, one contains batch normalization layers and the other does not, the behaviors of these two networks are then compared. Denote the loss functions as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {L}}}">

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {L}}}</annotation>

</semantics>

</math><img src="./c5914e99ad679b0a2d962ea00e6167a20ff0d4ce.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.583ex; height:2.843ex;" alt="{\displaystyle {\hat {L}}}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L}</annotation>

</semantics>

</math><img src="./103168b86f781fe6e9a4a87b8ea1cebe0ad4ede8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.583ex; height:2.176ex;" alt="{\displaystyle L}" loading="lazy">, respectively. Let the input to both networks be <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle x}</annotation>

</semantics>

</math><img src="./87f9e315fd7e2ba406057a97300593c4802b53e4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" loading="lazy">, and the output be <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y}</annotation>

</semantics>

</math><img src="./b8a6208ec717213d4317e666f1ae872e00620a0d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.155ex; height:2.009ex;" alt="{\displaystyle y}" loading="lazy">, for which <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y=Wx}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y=Wx}</annotation>

</semantics>

</math><img src="./17f279b02105527ae737f6379828d2ca382b69e0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:8.019ex; height:2.509ex;" alt="{\displaystyle y=Wx}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W}</annotation>

</semantics>

</math><img src="./54a9c4c547f4d6111f81946cad242b18298d70b7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.435ex; height:2.176ex;" alt="{\displaystyle W}" loading="lazy"> is the layer weights. For the second network, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y}</annotation>

</semantics>

</math><img src="./b8a6208ec717213d4317e666f1ae872e00620a0d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.155ex; height:2.009ex;" alt="{\displaystyle y}" loading="lazy"> additionally goes through a batch normalization layer. Denote the normalized activation as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {y}}}">

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {y}}}</annotation>

</semantics>

</math><img src="./3dc8de3d8ea01304329ef9518fad7a6d196c4c01.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.302ex; height:2.509ex;" alt="{\displaystyle {\hat {y}}}" loading="lazy">, which has zero mean and unit variance. Let the transformed activation be <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z=\gamma {\hat {y}}+\beta }">

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z=\gamma {\hat {y}}+\beta }</annotation>

</semantics>

</math><img src="./2cf9bf09cfb9fdb93161e82b93e61d1e1b724b6c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:10.924ex; height:2.676ex;" alt="{\displaystyle z=\gamma {\hat {y}}+\beta }" loading="lazy">, and suppose <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma }</annotation>

</semantics>

</math><img src="./a223c880b0ce3da8f64ee33c4f0010beee400b1a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.262ex; height:2.176ex;" alt="{\displaystyle \gamma }" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta }</annotation>

</semantics>

</math><img src="./7ed48a5e36207156fb792fa79d29925d2f7901e8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.332ex; height:2.509ex;" alt="{\displaystyle \beta }" loading="lazy"> are constants. Finally, denote the standard deviation over a mini-batch <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {y_{j}}}\in \mathbb {R} ^{m}}">

<mover>

<msub>

</mrow>

</msub>

</mover>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {y_{j}}}\in \mathbb {R} ^{m}}</annotation>

</semantics>

</math><img src="./a8527f733bd4c01bd2aeb0b1cb8eecd026e27649.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:8.243ex; height:3.009ex;" alt="{\displaystyle {\hat {y_{j}}}\in \mathbb {R} ^{m}}" loading="lazy"> as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma _{j}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma _{j}}</annotation>

</semantics>

</math><img src="./a234b1f289568934f127c2dd68ba77b6ef3f3569.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:2.237ex; height:2.343ex;" alt="{\displaystyle \sigma _{j}}" loading="lazy">.

First, it can be shown that the gradient magnitude of a batch normalized network, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle ||\triangledown _{y_{i}}{\hat {L}}||}">

</mrow>

</mrow>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle ||\triangledown _{y_{i}}{\hat {L}}||}</annotation>

</semantics>

</math><img src="./47443c58f1c9ffb4509b6688b28030172e35d79e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:7.511ex; height:3.509ex;" alt="{\displaystyle ||\triangledown _{y_{i}}{\hat {L}}||}" loading="lazy">, is bounded, with the bound expressed as

</mrow>

</mrow>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<mfrac>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</msubsup>

</mfrac>

</mrow>

</mrow>

</mrow>

</mrow>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<mfrac>

</mfrac>

</mrow>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<mfrac>

</mfrac>

</mrow>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle ||\triangledown _{y_{i}}{\hat {L}}||^{2}\leq {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}{\Bigg (}||\triangledown _{y_{i}}L||^{2}-{\frac {1}{m}}\langle 1,\triangledown _{y_{i}}L\rangle ^{2}-{\frac {1}{m}}\langle \triangledown _{y_{i}}L,{\hat {y}}_{j}\rangle ^{2}{\bigg )}}</annotation>

</semantics>

</math><img src="./ba3c5121143fe7afbff0230f491dba07a1c02c61.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:59.451ex; height:7.509ex;" alt="{\displaystyle ||\triangledown _{y_{i}}{\hat {L}}||^{2}\leq {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}{\Bigg (}||\triangledown _{y_{i}}L||^{2}-{\frac {1}{m}}\langle 1,\triangledown _{y_{i}}L\rangle ^{2}-{\frac {1}{m}}\langle \triangledown _{y_{i}}L,{\hat {y}}_{j}\rangle ^{2}{\bigg )}}" loading="lazy">.

Since the gradient magnitude represents the Lipschitzness of the loss, this relationship indicates that a batch normalized network could achieve greater Lipschitzness comparatively. Notice that the bound gets tighter when the gradient <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \triangledown _{y_{i}}{\hat {L}}}">

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \triangledown _{y_{i}}{\hat {L}}}</annotation>

</semantics>

</math><img src="./a386fa35b81502006040e9ea35d424dd997b71cf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:4.924ex; height:3.509ex;" alt="{\displaystyle \triangledown _{y_{i}}{\hat {L}}}" loading="lazy"> correlates with the activation <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {y_{i}}}}">

<mover>

<msub>

</mrow>

</msub>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {y_{i}}}}</annotation>

</semantics>

</math><img src="./84e9f46b496de85491c3227866df492db14c09fb.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.939ex; height:2.509ex;" alt="{\displaystyle {\hat {y_{i}}}}" loading="lazy">, which is a common phenomena. The scaling of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}}">

<mfrac>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</msubsup>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}}</annotation>

</semantics>

</math><img src="./eee6c2b2c2b869c353530cac7ed593c6fadd0ed1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:3.221ex; height:7.009ex;" alt="{\displaystyle {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}}" loading="lazy"> is also significant, since the variance is often large.

Secondly, the quadratic form of the loss Hessian with respect to activation in the gradient direction can be bounded as

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<msup>

</mrow>

</msup>

<mfrac>

<mrow>

<mover>

</mover>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<mfrac>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mfrac>

</mrow>

</mrow>

<mfrac>

<mrow>

<mover>

</mover>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

<mfrac>

<mrow>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

<mfrac>

<mrow>

<mover>

</mover>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mrow>

<mfrac>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<mover>

<msub>

</mrow>

</msub>

</mover>

</mrow>

</mrow>

</mrow>

<mfrac>

<mrow>

<mover>

</mover>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (\triangledown _{y_{j}}{\hat {L}})^{T}{\frac {\partial {\hat {L}}}{\partial y_{j}\partial y_{j}}}(\triangledown _{y_{j}}{\hat {L}})\leq {\frac {\gamma ^{2}}{\sigma ^{2}}}{\bigg (}{\frac {\partial {\hat {L}}}{\partial y_{j}}}{\bigg )}^{T}{\bigg (}{\frac {\partial L}{\partial y_{j}\partial y_{j}}}{\bigg )}{\bigg (}{\frac {\partial {\hat {L}}}{\partial y_{j}}}{\bigg )}-{\frac {\gamma }{m\sigma ^{2}}}\langle \triangledown _{y_{j}}L,{\hat {y_{j}}}\rangle {\bigg |}{\bigg |}{\frac {\partial {\hat {L}}}{\partial y_{j}}}{\bigg |}{\bigg |}^{2}}</annotation>

</semantics>

</math><img src="./598ba229b7a7a25cdeaf500a84d1b8c1fdda5fc8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:82.404ex; height:6.843ex;" alt="{\displaystyle (\triangledown _{y_{j}}{\hat {L}})^{T}{\frac {\partial {\hat {L}}}{\partial y_{j}\partial y_{j}}}(\triangledown _{y_{j}}{\hat {L}})\leq {\frac {\gamma ^{2}}{\sigma ^{2}}}{\bigg (}{\frac {\partial {\hat {L}}}{\partial y_{j}}}{\bigg )}^{T}{\bigg (}{\frac {\partial L}{\partial y_{j}\partial y_{j}}}{\bigg )}{\bigg (}{\frac {\partial {\hat {L}}}{\partial y_{j}}}{\bigg )}-{\frac {\gamma }{m\sigma ^{2}}}\langle \triangledown _{y_{j}}L,{\hat {y_{j}}}\rangle {\bigg |}{\bigg |}{\frac {\partial {\hat {L}}}{\partial y_{j}}}{\bigg |}{\bigg |}^{2}}" loading="lazy">.

The scaling of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}}">

<mfrac>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</msubsup>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}}</annotation>

</semantics>

</math><img src="./eee6c2b2c2b869c353530cac7ed593c6fadd0ed1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:3.221ex; height:7.009ex;" alt="{\displaystyle {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}}" loading="lazy"> indicates that the loss Hessian is resilient to the mini-batch variance, whereas the second term on the right hand side suggests that it becomes smoother when the <a href="Hessian_matrix" title="Hessian matrix">Hessian</a> and the inner product are non-negative. If the loss is locally <a href="Convex_function" title="Convex function">convex</a>, then the Hessian is <a href="Positive_semidefinite_matrix" class="mw-redirect" title="Positive semidefinite matrix">positive semi-definite</a>, while the inner product is positive if <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {g_{j}}}}">

<mover>

<msub>

</mrow>

</msub>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {g_{j}}}}</annotation>

</semantics>

</math><img src="./cbd3bc5d8e75dc374257ba8018dd9f23476ee33c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:2.019ex; height:2.843ex;" alt="{\displaystyle {\hat {g_{j}}}}" loading="lazy"> is in the direction towards the minimum of the loss. It could thus be concluded from this inequality that the gradient generally becomes more predictive with the batch normalization layer.

It then follows to translate the bounds related to the loss with respect to the normalized activation to a bound on the loss with respect to the network weights:

<mover>

<msub>

</mrow>

</msub>

</mover>

</mrow>

<mfrac>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</msubsup>

</mfrac>

</mrow>

</mrow>

</mrow>

</msubsup>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</msubsup>

<msup>

</mrow>

</msup>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {g_{j}}}\leq {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}(g_{j}^{2}-m\mu _{g_{j}}^{2}-\lambda ^{2}\langle \triangledown _{y_{j}}L,{\hat {y}}_{j}\rangle ^{2})}</annotation>

</semantics>

</math><img src="./c94b8189deaf786bb78f83dcb51f2bc422cfffbc.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.171ex; width:36.705ex; height:7.009ex;" alt="{\displaystyle {\hat {g_{j}}}\leq {\frac {\gamma ^{2}}{\sigma _{j}^{2}}}(g_{j}^{2}-m\mu _{g_{j}}^{2}-\lambda ^{2}\langle \triangledown _{y_{j}}L,{\hat {y}}_{j}\rangle ^{2})}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle g_{j}=max_{||X||\leq \lambda }||\triangledown _{W}L||^{2}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</mrow>

</mrow>

</mrow>

</msub>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle g_{j}=max_{||X||\leq \lambda }||\triangledown _{W}L||^{2}}</annotation>

</semantics>

</math><img src="./5f76d93a70a9de73a91da4c44769bc7a720c3bb0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:24.273ex; height:3.676ex;" alt="{\displaystyle g_{j}=max_{||X||\leq \lambda }||\triangledown _{W}L||^{2}}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {g}}_{j}=max_{||X||\leq \lambda }||\triangledown _{W}{\hat {L}}||^{2}}">

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</mrow>

</mrow>

</mrow>

</msub>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {g}}_{j}=max_{||X||\leq \lambda }||\triangledown _{W}{\hat {L}}||^{2}}</annotation>

</semantics>

</math><img src="./a1c5032bb8d3a7aa61a390c748e95ada0599d670.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:24.395ex; height:3.676ex;" alt="{\displaystyle {\hat {g}}_{j}=max_{||X||\leq \lambda }||\triangledown _{W}{\hat {L}}||^{2}}" loading="lazy">.

In addition to the smoother landscape, it is further shown that batch normalization could result in a better initialization with the following inequality:

</mrow>

</mrow>

<msub>

</mrow>

</msub>

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<mfrac>

<mrow>

</mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle ||W_{0}-{\hat {W}}^{*}||^{2}\leq ||W_{0}-W^{*}||^{2}-{\frac {1}{||W^{*}||^{2}}}(||W^{*}||^{2}-\langle W^{*},W_{0}\rangle )^{2}}</annotation>

</semantics>

</math><img src="./9ac3bd274bf4ea26d42044bf5f0f1fa130174fa6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:63.051ex; height:6.343ex;" alt="{\displaystyle ||W_{0}-{\hat {W}}^{*}||^{2}\leq ||W_{0}-W^{*}||^{2}-{\frac {1}{||W^{*}||^{2}}}(||W^{*}||^{2}-\langle W^{*},W_{0}\rangle )^{2}}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W^{*}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W^{*}}</annotation>

</semantics>

</math><img src="./5292cb3dbf333bb0b51f39fb9f724fa8d4dfcc63.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.562ex; height:2.343ex;" alt="{\displaystyle W^{*}}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {W}}^{*}}">

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {W}}^{*}}</annotation>

</semantics>

</math><img src="./a9f598335a95fd9cc1d7eb039fd9ca278165a96c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.489ex; height:3.009ex;" alt="{\displaystyle {\hat {W}}^{*}}" loading="lazy"> are the local optimal weights for the two networks, respectively.

Some scholars argue that the above analysis cannot fully capture the performance of batch normalization, because the proof only concerns the largest eigenvalue, or equivalently, one direction in the landscape at all points. It is suggested that the complete eigenspectrum needs to be taken into account to make a conclusive analysis.<a href="#cite_note-:2-8">[8]</a><a href="#cite_note-:1-5">[5]</a>

<div class="mw-heading mw-heading3"><h3 id="Measure">Measure</h3></div>

Since it is hypothesized that batch normalization layers could reduce internal covariate shift, an experiment is set up to measure quantitatively how much covariate shift is reduced. First, the notion of internal covariate shift needs to be defined mathematically. Specifically, to quantify the adjustment that a layer's parameters make in response to updates in previous layers, the correlation between the gradients of the loss before and after all previous layers are updated is measured, since gradients could capture the shifts from the first-order training method. If the shift introduced by the changes in previous layers is small, then the correlation between the gradients would be close to 1.

The correlation between the gradients are computed for four models: a standard VGG network,<a href="#cite_note-VGG-6">[6]</a> a VGG network with batch normalization layers, a 25-layer deep linear network (DLN) trained with full-batch gradient descent, and a DLN network with batch normalization layers. Interestingly, it is shown that the standard VGG and DLN models both have higher correlations of gradients compared with their counterparts, indicating that the additional batch normalization layers are not reducing internal covariate shift.

<div class="mw-heading mw-heading3"><h3 id="Vanishing/exploding_gradients">Vanishing/exploding gradients</h3></div>

Even though batchnorm was originally introduced to alleviate <a href="Vanishing_gradient_problem" title="Vanishing gradient problem">gradient vanishing or explosion problems</a>, a deep batchnorm network in fact suffers from gradient explosion at initialization time, no matter what it uses for nonlinearity. Thus the optimization landscape is very far from smooth for a randomly initialized, deep batchnorm network.

More precisely, if the network has <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L}</annotation>

</semantics>

</math><img src="./103168b86f781fe6e9a4a87b8ea1cebe0ad4ede8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.583ex; height:2.176ex;" alt="{\displaystyle L}" loading="lazy"> layers, then the gradient of the first layer weights has norm <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle >c\lambda ^{L}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle >c\lambda ^{L}}</annotation>

</semantics>

</math><img src="./cab76f58a7a5dccf0f9ed0c5d78e1db2fb2f7435.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.167ex; height:2.676ex;" alt="{\displaystyle >c\lambda ^{L}}" loading="lazy"> for some <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \lambda >1,c>0}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \lambda >1,c>0}</annotation>

</semantics>

</math><img src="./afd304b28687ea62f39b2a8fc2fb7916aa8eb531.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:11.918ex; height:2.509ex;" alt="{\displaystyle \lambda >1,c>0}" loading="lazy"> depending only on the nonlinearity.

For any fixed nonlinearity, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \lambda }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \lambda }</annotation>

</semantics>

</math><img src="./b43d0ea3c9c025af1be9128e62a18fa74bedda2a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.355ex; height:2.176ex;" alt="{\displaystyle \lambda }" loading="lazy"> decreases as the batch size increases. For example, for ReLU, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \lambda }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \lambda }</annotation>

</semantics>

</math><img src="./b43d0ea3c9c025af1be9128e62a18fa74bedda2a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.355ex; height:2.176ex;" alt="{\displaystyle \lambda }" loading="lazy"> decreases to <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \pi /(\pi -1)\approx 1.467}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \pi /(\pi -1)\approx 1.467}</annotation>

</semantics>

</math><img src="./6e3922a4b656e6d4281f61c029cb1c8b0943cc72.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:18.034ex; height:2.843ex;" alt="{\displaystyle \pi /(\pi -1)\approx 1.467}" loading="lazy"> as the batch size tends to infinity.

Practically, this means deep batchnorm networks are untrainable.

This is only relieved by skip connections in the fashion of residual networks.<a href="#cite_note-:7-9">[9]</a>

This gradient explosion on the surface contradicts the smoothness property explained in the previous section, but in fact they are consistent. The previous section studies the effect of inserting a single batchnorm in a network, while the gradient explosion depends on stacking batchnorms typical of modern deep neural networks.

<div class="mw-heading mw-heading3"><h3 id="Decoupling">Decoupling</h3></div>

Another possible reason for the success of batch normalization is that it decouples the length and direction of the weight vectors and thus facilitates better training.

By interpreting batch norm as a reparametrization of weight space, it can be shown that the length and the direction of the weights are separated and can thus be trained separately. For a particular neural network unit with input <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle x}</annotation>

</semantics>

</math><img src="./87f9e315fd7e2ba406057a97300593c4802b53e4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" loading="lazy"> and weight vector <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w}</annotation>

</semantics>

</math><img src="./88b1e0c8e1be5ebe69d18a8010676fa42d7961e6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.664ex; height:1.676ex;" alt="{\displaystyle w}" loading="lazy">, denote its output as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f(w)=E_{x}[\phi (x^{T}w)]}">

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f(w)=E_{x}[\phi (x^{T}w)]}</annotation>

</semantics>

</math><img src="./870188d380627f799e598e056ea71b4c23815741.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.61ex; height:3.176ex;" alt="{\displaystyle f(w)=E_{x}[\phi (x^{T}w)]}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \phi }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \phi }</annotation>

</semantics>

</math><img src="./72b1f30316670aee6270a28334bdf4f5072cdde4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.385ex; height:2.509ex;" alt="{\displaystyle \phi }" loading="lazy"> is the activation function, and denote <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle S=E[xx^{T}]}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle S=E[xx^{T}]}</annotation>

</semantics>

</math><img src="./71f3a47dfa9441a4050edac91329e66ac747684b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.716ex; height:3.176ex;" alt="{\displaystyle S=E[xx^{T}]}" loading="lazy">. Assume that <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle E[x]=0}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle E[x]=0}</annotation>

</semantics>

</math><img src="./f464a09bd74ae67c6e8094c223aa1edd093699c9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.66ex; height:2.843ex;" alt="{\displaystyle E[x]=0}" loading="lazy">, and that the spectrum of the matrix <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle S}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle S}</annotation>

</semantics>

</math><img src="./4611d85173cd3b508e67077d4a1252c9c05abca2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.499ex; height:2.176ex;" alt="{\displaystyle S}" loading="lazy"> is bounded as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 0<\mu =\lambda _{min}(S)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 0<\mu =\lambda _{min}(S)}</annotation>

</semantics>

</math><img src="./c780462af127d32b872a60841bd24c25da8ea6d9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:16.653ex; height:2.843ex;" alt="{\displaystyle 0<\mu =\lambda _{min}(S)}" loading="lazy">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L=\lambda _{max}(S)<\infty }">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L=\lambda _{max}(S)<\infty }</annotation>

</semantics>

</math><img src="./d3c7406de80618dc4081e9c8881339b56595e992.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:18.252ex; height:2.843ex;" alt="{\displaystyle L=\lambda _{max}(S)<\infty }" loading="lazy">, such that <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle S}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle S}</annotation>

</semantics>

</math><img src="./4611d85173cd3b508e67077d4a1252c9c05abca2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.499ex; height:2.176ex;" alt="{\displaystyle S}" loading="lazy"> is symmetric positive definite. Adding batch normalization to this unit thus results in

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

<mfrac>

<mrow>

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f_{BN}(w,\gamma ,\beta )=E_{x}[\phi (BN(x^{T}w))]=E_{x}{\bigg [}\phi {\bigg (}\gamma ({\frac {x^{T}w-E_{x}[x^{T}w]}{var_{x}[x^{T}w]^{1/2}}})+\beta {\bigg )}{\bigg ]}}</annotation>

</semantics>

</math><img src="./ef3c8776e25fec4bb27c4e556fafeda7d35686a3.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.838ex; width:69.824ex; height:6.843ex;" alt="{\displaystyle f_{BN}(w,\gamma ,\beta )=E_{x}[\phi (BN(x^{T}w))]=E_{x}{\bigg [}\phi {\bigg (}\gamma ({\frac {x^{T}w-E_{x}[x^{T}w]}{var_{x}[x^{T}w]^{1/2}}})+\beta {\bigg )}{\bigg ]}}" loading="lazy">, by definition.

The variance term can be simplified such that <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle var_{x}[x^{T}w]=w^{T}Sw}">

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle var_{x}[x^{T}w]=w^{T}Sw}</annotation>

</semantics>

</math><img src="./64581f03bf8607566e4f8126ce8822e23ca4454d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.571ex; height:3.176ex;" alt="{\displaystyle var_{x}[x^{T}w]=w^{T}Sw}" loading="lazy">. Assume that <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle x}</annotation>

</semantics>

</math><img src="./87f9e315fd7e2ba406057a97300593c4802b53e4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.33ex; height:1.676ex;" alt="{\displaystyle x}" loading="lazy"> has zero mean and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta }</annotation>

</semantics>

</math><img src="./7ed48a5e36207156fb792fa79d29925d2f7901e8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.332ex; height:2.509ex;" alt="{\displaystyle \beta }" loading="lazy"> can be omitted, then it follows that

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

<mfrac>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f_{BN}(w,\gamma )=E_{x}{\bigg [}\phi {\bigg (}\gamma {\frac {x^{T}w}{(w^{T}Sw)^{1/2}}}{\bigg )}{\bigg ]}}</annotation>

</semantics>

</math><img src="./8073fc1cb01c6722a53258ee44d8ad020f8b1e6f.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.838ex; width:35.918ex; height:6.676ex;" alt="{\displaystyle f_{BN}(w,\gamma )=E_{x}{\bigg [}\phi {\bigg (}\gamma {\frac {x^{T}w}{(w^{T}Sw)^{1/2}}}{\bigg )}{\bigg ]}}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (w^{T}Sw)^{\frac {1}{2}}}">

<msup>

</mrow>

</msup>

<msup>

<mfrac>

</mfrac>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (w^{T}Sw)^{\frac {1}{2}}}</annotation>

</semantics>

</math><img src="./83a8fad691375fdec2a94f2a066f155172c6045b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.762ex; height:4.009ex;" alt="{\displaystyle (w^{T}Sw)^{\frac {1}{2}}}" loading="lazy"> is the induced norm of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle S}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle S}</annotation>

</semantics>

</math><img src="./4611d85173cd3b508e67077d4a1252c9c05abca2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.499ex; height:2.176ex;" alt="{\displaystyle S}" loading="lazy">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle ||w||_{s}}">

</mrow>

</mrow>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle ||w||_{s}}</annotation>

</semantics>

</math><img src="./d0357dc403927cc26418f5ccbe598f35d7b65606.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:5.255ex; height:3.009ex;" alt="{\displaystyle ||w||_{s}}" loading="lazy">.

Hence, it could be concluded that <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{BN}(w,\gamma )=E_{x}[\phi (x^{T}{\tilde {w}})]}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f_{BN}(w,\gamma )=E_{x}[\phi (x^{T}{\tilde {w}})]}</annotation>

</semantics>

</math><img src="./0479b5ef1b32a46c9837a35b2e6746a8a8b36803.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:24.705ex; height:3.176ex;" alt="{\displaystyle f_{BN}(w,\gamma )=E_{x}[\phi (x^{T}{\tilde {w}})]}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tilde {w}}=\gamma {\frac {w}{||w||_{s}}}}">

<mover>

</mover>

</mrow>

<mfrac>

<mrow>

</mrow>

</mrow>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {w}}=\gamma {\frac {w}{||w||_{s}}}}</annotation>

</semantics>

</math><img src="./4fb18955e26cd1b8f38355e59f34f94b4dff6b2e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:12.116ex; height:5.509ex;" alt="{\displaystyle {\tilde {w}}=\gamma {\frac {w}{||w||_{s}}}}" loading="lazy">, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma }</annotation>

</semantics>

</math><img src="./a223c880b0ce3da8f64ee33c4f0010beee400b1a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.262ex; height:2.176ex;" alt="{\displaystyle \gamma }" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w}</annotation>

</semantics>

</math><img src="./88b1e0c8e1be5ebe69d18a8010676fa42d7961e6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.664ex; height:1.676ex;" alt="{\displaystyle w}" loading="lazy"> accounts for its length and direction separately. This property could then be used to prove the faster convergence of problems with batch normalization.

<div class="mw-heading mw-heading2"><h2 id="Linear_convergence">Linear convergence</h2></div>

<div class="mw-heading mw-heading3"><h3 id="Least-square_problem">Least-square problem</h3></div>

With the reparametrization interpretation, it could then be proved that applying batch normalization to the ordinary least squares problem achieves a linear convergence rate in gradient descent, which is faster than the regular gradient descent with only sub-linear convergence.

Denote the objective of minimizing an ordinary least squares problem as

<msub>

<mover>

</mover>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<msub>

<mover>

</mover>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<mover>

</mover>

</mrow>

<msup>

</mrow>

</msup>

<msub>

<mover>

</mover>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<mover>

</mover>

</mrow>

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle min_{{\tilde {w}}\in R^{d}}f_{OLS}({\tilde {w}})=min_{{\tilde {w}}\in R^{d}}(E_{x,y}[(y-x^{T}{\tilde {w}})^{2}])=min_{{\tilde {w}}\in R^{d}}(2u^{T}{\tilde {w}}+{\tilde {w}}^{T}S{\tilde {w}})}</annotation>

</semantics>

</math><img src="./1efc5773164ae155830adf89896a7cf642e3359a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:76ex; height:3.343ex;" alt="{\displaystyle min_{{\tilde {w}}\in R^{d}}f_{OLS}({\tilde {w}})=min_{{\tilde {w}}\in R^{d}}(E_{x,y}[(y-x^{T}{\tilde {w}})^{2}])=min_{{\tilde {w}}\in R^{d}}(2u^{T}{\tilde {w}}+{\tilde {w}}^{T}S{\tilde {w}})}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle u=E[-yx]}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle u=E[-yx]}</annotation>

</semantics>

</math><img src="./946686cd36bcf56c7210946d1a01c17c92c146e1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.791ex; height:2.843ex;" alt="{\displaystyle u=E[-yx]}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle S=E[xx^{T}]}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle S=E[xx^{T}]}</annotation>

</semantics>

</math><img src="./71f3a47dfa9441a4050edac91329e66ac747684b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.716ex; height:3.176ex;" alt="{\displaystyle S=E[xx^{T}]}" loading="lazy">.

Since <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tilde {w}}=\gamma {\frac {w}{||w||_{s}}}}">

<mover>

</mover>

</mrow>

<mfrac>

<mrow>

</mrow>

</mrow>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {w}}=\gamma {\frac {w}{||w||_{s}}}}</annotation>

</semantics>

</math><img src="./4fb18955e26cd1b8f38355e59f34f94b4dff6b2e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:12.116ex; height:5.509ex;" alt="{\displaystyle {\tilde {w}}=\gamma {\frac {w}{||w||_{s}}}}" loading="lazy">, the objective thus becomes

<msub>

<msup>

</mrow>

</msup>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

<msup>

</mrow>

</msup>

</mrow>

</msub>

</mrow>

<mfrac>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

</mrow>

</mrow>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle min_{w\in R^{d}\backslash \{0\},\gamma \in R}f_{OLS}(w,\gamma )=min_{w\in R^{d}\backslash \{0\},\gamma \in R}{\bigg (}2\gamma {\frac {u^{T}w}{||w||_{S}+\gamma ^{2}}}{\bigg )}}</annotation>

</semantics>

</math><img src="./df2b7bb3436b083936de16ca1a8a8feb52874403.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.838ex; width:62.741ex; height:6.676ex;" alt="{\displaystyle min_{w\in R^{d}\backslash \{0\},\gamma \in R}f_{OLS}(w,\gamma )=min_{w\in R^{d}\backslash \{0\},\gamma \in R}{\bigg (}2\gamma {\frac {u^{T}w}{||w||_{S}+\gamma ^{2}}}{\bigg )}}" loading="lazy">, where 0 is excluded to avoid 0 in the denominator.

Since the objective is convex with respect to <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma }</annotation>

</semantics>

</math><img src="./a223c880b0ce3da8f64ee33c4f0010beee400b1a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.262ex; height:2.176ex;" alt="{\displaystyle \gamma }" loading="lazy">, its optimal value could be calculated by setting the partial derivative of the objective against <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma }</annotation>

</semantics>

</math><img src="./a223c880b0ce3da8f64ee33c4f0010beee400b1a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.262ex; height:2.176ex;" alt="{\displaystyle \gamma }" loading="lazy"> to 0. The objective could be further simplified to be

<msub>

<msup>

</mrow>

</msup>

</mrow>

</msub>

<msub>

<msup>

</mrow>

</msup>

</mrow>

</msub>

</mrow>

<mfrac>

<mrow>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle min_{w\in R^{d}\backslash \{0\}}\rho (w)=min_{w\in R^{d}\backslash \{0\}}{\bigg (}-{\frac {w^{T}uu^{T}w}{w^{T}Sw}}{\bigg )}}</annotation>

</semantics>

</math><img src="./6229111c77246ba81ee560715024f769f95b542b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:47.918ex; height:6.343ex;" alt="{\displaystyle min_{w\in R^{d}\backslash \{0\}}\rho (w)=min_{w\in R^{d}\backslash \{0\}}{\bigg (}-{\frac {w^{T}uu^{T}w}{w^{T}Sw}}{\bigg )}}" loading="lazy">.

Note that this objective is a form of the generalized Rayleigh quotient

<mover>

</mover>

</mrow>

<mfrac>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {\rho }}(w)={\frac {w^{T}Bw}{w^{T}Aw}}}</annotation>

</semantics>

</math><img src="./68c6543e66c85166484c956afb65e16dd32fed82.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.171ex; width:15.265ex; height:6.009ex;" alt="{\displaystyle {\tilde {\rho }}(w)={\frac {w^{T}Bw}{w^{T}Aw}}}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle B\in R^{d\times d}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle B\in R^{d\times d}}</annotation>

</semantics>

</math><img src="./a365fbacda143d5138879474fe8b9dd74ce8b2c9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:9.599ex; height:2.676ex;" alt="{\displaystyle B\in R^{d\times d}}" loading="lazy"> is a symmetric matrix and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle A\in R^{d\times d}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle A\in R^{d\times d}}</annotation>

</semantics>

</math><img src="./8de279512674b95aa9d64c8dbf34d020710f8ed1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:9.578ex; height:2.676ex;" alt="{\displaystyle A\in R^{d\times d}}" loading="lazy"> is a symmetric <a href="Positive_definiteness" title="Positive definiteness">positive definite</a> matrix.

It is proven that the gradient descent convergence rate of the generalized <a href="Rayleigh_quotient" title="Rayleigh quotient">Rayleigh quotient</a> is

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mrow>

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\lambda _{1}-\rho (w_{t+1})}{\rho (w_{t+1}-\lambda _{2})}}\leq {\bigg (}1-{\frac {\lambda _{1}-\lambda _{2}}{\lambda _{1}-\lambda _{min}}}{\bigg )}^{2t}{\frac {\lambda _{1}-\rho (w_{t})}{\rho (w_{t})-\lambda _{2}}}}</annotation>

</semantics>

</math><img src="./a894c98e578963723b287726f9b223075d38e4b7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:48.116ex; height:6.676ex;" alt="{\displaystyle {\frac {\lambda _{1}-\rho (w_{t+1})}{\rho (w_{t+1}-\lambda _{2})}}\leq {\bigg (}1-{\frac {\lambda _{1}-\lambda _{2}}{\lambda _{1}-\lambda _{min}}}{\bigg )}^{2t}{\frac {\lambda _{1}-\rho (w_{t})}{\rho (w_{t})-\lambda _{2}}}}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \lambda _{1}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \lambda _{1}}</annotation>

</semantics>

</math><img src="./571a423bece8f29bcd1b48572f18dd4f6213dce2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.409ex; height:2.509ex;" alt="{\displaystyle \lambda _{1}}" loading="lazy"> is the largest <a href="Eigenvalues_and_eigenvectors" title="Eigenvalues and eigenvectors">eigenvalue</a> of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle B}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle B}</annotation>

</semantics>

</math><img src="./47136aad860d145f75f3eed3022df827cee94d7a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.764ex; height:2.176ex;" alt="{\displaystyle B}" loading="lazy">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \lambda _{2}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \lambda _{2}}</annotation>

</semantics>

</math><img src="./6b668a1bd1e8ab9452ca975b7497546e7c1ba187.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.409ex; height:2.509ex;" alt="{\displaystyle \lambda _{2}}" loading="lazy"> is the second largest eigenvalue of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle B}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle B}</annotation>

</semantics>

</math><img src="./47136aad860d145f75f3eed3022df827cee94d7a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.764ex; height:2.176ex;" alt="{\displaystyle B}" loading="lazy">, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \lambda _{min}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \lambda _{min}}</annotation>

</semantics>

</math><img src="./9ea5700b1cdbc0c935d3d34bc03c778a8641cc79.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.584ex; height:2.509ex;" alt="{\displaystyle \lambda _{min}}" loading="lazy"> is the smallest eigenvalue of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle B}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle B}</annotation>

</semantics>

</math><img src="./47136aad860d145f75f3eed3022df827cee94d7a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.764ex; height:2.176ex;" alt="{\displaystyle B}" loading="lazy">.<a href="#cite_note-10">[10]</a>

In our case, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle B=uu^{T}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle B=uu^{T}}</annotation>

</semantics>

</math><img src="./c8158101fea0192702480f5231aba7e185185789.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:8.911ex; height:2.676ex;" alt="{\displaystyle B=uu^{T}}" loading="lazy"> is a rank one matrix, and the convergence result can be simplified accordingly. Specifically, consider gradient descent steps of the form <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w_{t+1}=w_{t}-\eta _{t}\triangledown \rho (w_{t})}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w_{t+1}=w_{t}-\eta _{t}\triangledown \rho (w_{t})}</annotation>

</semantics>

</math><img src="./71855c87dab86bc52e1e722e6b7dc695c7891b66.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:22.18ex; height:2.843ex;" alt="{\displaystyle w_{t+1}=w_{t}-\eta _{t}\triangledown \rho (w_{t})}" loading="lazy"> with step size <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \eta _{t}={\frac {w_{t}^{T}Sw_{t}}{2L|\rho (w_{t})|}}}">

<msub>

</mrow>

</msub>

<mfrac>

<mrow>

</mrow>

</mrow>

</msubsup>

<msub>

</mrow>

</msub>

</mrow>

<mrow>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \eta _{t}={\frac {w_{t}^{T}Sw_{t}}{2L|\rho (w_{t})|}}}</annotation>

</semantics>

</math><img src="./4e30e4bf8005a6bfbc82ab34fc4ecef53a3e109d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:15.456ex; height:6.676ex;" alt="{\displaystyle \eta _{t}={\frac {w_{t}^{T}Sw_{t}}{2L|\rho (w_{t})|}}}" loading="lazy">, and starting from <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \rho (w_{0})\neq 0}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \rho (w_{0})\neq 0}</annotation>

</semantics>

</math><img src="./e52618bcfa5e5a8170f3f52c9c629aad11a788ea.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.991ex; height:2.843ex;" alt="{\displaystyle \rho (w_{0})\neq 0}" loading="lazy">, then

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mrow>

<mfrac>

</mfrac>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \rho (w_{t})-\rho (w^{*})\leq {\bigg (}1-{\frac {\mu }{L}}{\bigg )}^{2t}(\rho (w_{0})-\rho (w^{*}))}</annotation>

</semantics>

</math><img src="./8d977d55687351b9820910c4b55adcce3c3fde06.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:44.77ex; height:6.509ex;" alt="{\displaystyle \rho (w_{t})-\rho (w^{*})\leq {\bigg (}1-{\frac {\mu }{L}}{\bigg )}^{2t}(\rho (w_{0})-\rho (w^{*}))}" loading="lazy">.

<div class="mw-heading mw-heading3"><h3 id="Learning_halfspace_problem">Learning halfspace problem</h3></div>

The problem of learning halfspaces refers to the training of the <a href="Perceptron" title="Perceptron">Perceptron</a>, which is the simplest form of neural network. The optimization problem in this case is

<msub>

<mover>

</mover>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle min_{{\tilde {w}}\in R^{d}}f_{LH}({\tilde {w}})=E_{y,x}[\phi (z^{T}{\tilde {w}})]}</annotation>

</semantics>

</math><img src="./680bf7fc32309d376e4b6ee64086c2f40acd2e80.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:32.185ex; height:3.343ex;" alt="{\displaystyle min_{{\tilde {w}}\in R^{d}}f_{LH}({\tilde {w}})=E_{y,x}[\phi (z^{T}{\tilde {w}})]}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z=-yx}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z=-yx}</annotation>

</semantics>

</math><img src="./2f6325883caf5034a3039aa57b80ff90cf19508a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:8.48ex; height:2.343ex;" alt="{\displaystyle z=-yx}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \phi }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \phi }</annotation>

</semantics>

</math><img src="./72b1f30316670aee6270a28334bdf4f5072cdde4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.385ex; height:2.509ex;" alt="{\displaystyle \phi }" loading="lazy"> is an arbitrary loss function.

Suppose that <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \phi }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \phi }</annotation>

</semantics>

</math><img src="./72b1f30316670aee6270a28334bdf4f5072cdde4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.385ex; height:2.509ex;" alt="{\displaystyle \phi }" loading="lazy"> is infinitely differentiable and has a bounded derivative. Assume that the objective function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{LH}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f_{LH}}</annotation>

</semantics>

</math><img src="./94defc1ff6a3119cb74d8743797f2a9b5055b231.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.95ex; height:2.509ex;" alt="{\displaystyle f_{LH}}" loading="lazy"> is <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \zeta }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \zeta }</annotation>

</semantics>

</math><img src="./d5c3916703cae7938143d38865f78f27faadd4ae.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.095ex; height:2.509ex;" alt="{\displaystyle \zeta }" loading="lazy">-<a href="Smoothness" title="Smoothness">smooth</a>, and that a solution <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \alpha ^{*}=argmin_{\alpha }||\triangledown f(\alpha w)||^{2}}">

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \alpha ^{*}=argmin_{\alpha }||\triangledown f(\alpha w)||^{2}}</annotation>

</semantics>

</math><img src="./6ba3acfacaaef16df296a3a4d5c07b5e1866dc0a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:26.116ex; height:3.343ex;" alt="{\displaystyle \alpha ^{*}=argmin_{\alpha }||\triangledown f(\alpha w)||^{2}}" loading="lazy"> exists and is bounded such that <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle -\infty <\alpha ^{*}<\infty }">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle -\infty <\alpha ^{*}<\infty }</annotation>

</semantics>

</math><img src="./a7dcae0fdb761140b64f974e10d7515bf30dd776.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:15.194ex; height:2.509ex;" alt="{\displaystyle -\infty <\alpha ^{*}<\infty }" loading="lazy">. Also assume <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z}</annotation>

</semantics>

</math><img src="./bf368e72c009decd9b6686ee84a375632e11de98.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.088ex; height:1.676ex;" alt="{\displaystyle z}" loading="lazy"> is a <a href="Multivariate_normal_distribution" title="Multivariate normal distribution">multivariate normal random variable</a>. With the Gaussian assumption, it can be shown that all <a href="Critical_point_(network_science)" title="Critical point (network science)">critical points</a> lie on the same line, for any choice of loss function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \phi }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \phi }</annotation>

</semantics>

</math><img src="./72b1f30316670aee6270a28334bdf4f5072cdde4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.385ex; height:2.509ex;" alt="{\displaystyle \phi }" loading="lazy">. Specifically, the gradient of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{LH}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f_{LH}}</annotation>

</semantics>

</math><img src="./94defc1ff6a3119cb74d8743797f2a9b5055b231.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.95ex; height:2.509ex;" alt="{\displaystyle f_{LH}}" loading="lazy"> could be represented as

<msub>

<mover>

</mover>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \triangledown _{\tilde {w}}f_{LH}({\tilde {w}})=c_{1}({\tilde {w}})u+c_{2}({\tilde {w}})S{\tilde {w}}}</annotation>

</semantics>

</math><img src="./c2487deac441b8543d343361909d7d5986233409.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:32.027ex; height:2.843ex;" alt="{\displaystyle \triangledown _{\tilde {w}}f_{LH}({\tilde {w}})=c_{1}({\tilde {w}})u+c_{2}({\tilde {w}})S{\tilde {w}}}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle c_{1}({\tilde {w}})=E_{z}[\phi ^{(1)}(z^{T}{\tilde {w}})]-E_{z}[\phi ^{(2)}(z^{T}{\tilde {w}})](u^{T}{\tilde {w}})}">

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

<mover>

</mover>

</mrow>

<msup>

</mrow>

</msup>

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle c_{1}({\tilde {w}})=E_{z}[\phi ^{(1)}(z^{T}{\tilde {w}})]-E_{z}[\phi ^{(2)}(z^{T}{\tilde {w}})](u^{T}{\tilde {w}})}</annotation>

</semantics>

</math><img src="./6d41acf60dc403751d136a81ba607fbbf80b76ab.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:45.031ex; height:3.343ex;" alt="{\displaystyle c_{1}({\tilde {w}})=E_{z}[\phi ^{(1)}(z^{T}{\tilde {w}})]-E_{z}[\phi ^{(2)}(z^{T}{\tilde {w}})](u^{T}{\tilde {w}})}" loading="lazy">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle c_{2}({\tilde {w}})=E_{z}[\phi ^{(2)}(z^{T}{\tilde {w}})]}">

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle c_{2}({\tilde {w}})=E_{z}[\phi ^{(2)}(z^{T}{\tilde {w}})]}</annotation>

</semantics>

</math><img src="./e43fecb96d87a856bc20fe7e94d6f6fd172f58c3.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:22.316ex; height:3.343ex;" alt="{\displaystyle c_{2}({\tilde {w}})=E_{z}[\phi ^{(2)}(z^{T}{\tilde {w}})]}" loading="lazy">, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \phi ^{(i)}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \phi ^{(i)}}</annotation>

</semantics>

</math><img src="./7e06943f6204295a43850dac83b582d1c7de9aa1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.464ex; height:3.176ex;" alt="{\displaystyle \phi ^{(i)}}" loading="lazy"> is the <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i}</annotation>

</semantics>

</math><img src="./add78d8608ad86e54951b8c8bd6c8d8416533d20.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}" loading="lazy">-th derivative of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \phi }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \phi }</annotation>

</semantics>

</math><img src="./72b1f30316670aee6270a28334bdf4f5072cdde4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.385ex; height:2.509ex;" alt="{\displaystyle \phi }" loading="lazy">.

By setting the gradient to 0, it thus follows that the bounded critical points <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tilde {w}}_{*}}">

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {w}}_{*}}</annotation>

</semantics>

</math><img src="./92c500e6749b54d50218d0658abd54389e8c0c4d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.293ex; margin-bottom: -0.379ex; width:2.718ex; height:2.509ex;" alt="{\displaystyle {\tilde {w}}_{*}}" loading="lazy"> can be expressed as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tilde {w}}_{*}=g_{*}S^{-1}u}">

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {w}}_{*}=g_{*}S^{-1}u}</annotation>

</semantics>

</math><img src="./8010907b60d0727a6709bceaf0cd713e8dc529bb.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:13.164ex; height:3.009ex;" alt="{\displaystyle {\tilde {w}}_{*}=g_{*}S^{-1}u}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle g_{*}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle g_{*}}</annotation>

</semantics>

</math><img src="./158776589fce805bba6b36b91cf383832dfc50cf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.163ex; height:2.009ex;" alt="{\displaystyle g_{*}}" loading="lazy"> depends on <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tilde {w}}_{*}}">

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {w}}_{*}}</annotation>

</semantics>

</math><img src="./92c500e6749b54d50218d0658abd54389e8c0c4d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.293ex; margin-bottom: -0.379ex; width:2.718ex; height:2.509ex;" alt="{\displaystyle {\tilde {w}}_{*}}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \phi }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \phi }</annotation>

</semantics>

</math><img src="./72b1f30316670aee6270a28334bdf4f5072cdde4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.385ex; height:2.509ex;" alt="{\displaystyle \phi }" loading="lazy">. Combining this global property with length-direction decoupling, it could thus be proved that this optimization problem converges linearly.

First, a variation of <a href="Gradient_descent" title="Gradient descent">gradient descent</a> with batch normalization, Gradient Descent in Normalized Parameterization (GDNP), is designed for the objective function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle min_{w\in R^{d}\backslash \{0\},\gamma \in R}f_{LH}(w,\gamma )}">

<msub>

<msup>

</mrow>

</msup>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle min_{w\in R^{d}\backslash \{0\},\gamma \in R}f_{LH}(w,\gamma )}</annotation>

</semantics>

</math><img src="./8e4b2b83d6b5037fb923516552c6e7051c8d304e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.338ex; width:25.553ex; height:3.343ex;" alt="{\displaystyle min_{w\in R^{d}\backslash \{0\},\gamma \in R}f_{LH}(w,\gamma )}" loading="lazy">, such that the direction and length of the weights are updated separately. Denote the stopping criterion of GDNP as

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</msup>

<msup>

</mrow>

</msup>

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</msup>

<msup>

</mrow>

</msup>

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h(w_{t},\gamma _{t})=E_{z}[\phi '(z^{T}{\tilde {w}}_{t})](u^{T}w_{t})-E_{z}[\phi ''(z^{T}{\tilde {w}}_{t})](u^{T}w_{t})^{2}}</annotation>

</semantics>

</math><img src="./f5d9060d18d7abd003631c293bd29c2432de2d7c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:55.904ex; height:3.176ex;" alt="{\displaystyle h(w_{t},\gamma _{t})=E_{z}[\phi '(z^{T}{\tilde {w}}_{t})](u^{T}w_{t})-E_{z}[\phi ''(z^{T}{\tilde {w}}_{t})](u^{T}w_{t})^{2}}" loading="lazy">.

Let the step size be

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mfrac>

<mrow>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle s_{t}=s(w_{t},\gamma _{t})=-{\frac {||w_{t}||_{S}^{3}}{Lg_{t}h(w_{t},\gamma _{t})}}}</annotation>

</semantics>

</math><img src="./a8a8e930d6a9ce8f24fb27a2bdd552b417323fd8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:31.432ex; height:7.009ex;" alt="{\displaystyle s_{t}=s(w_{t},\gamma _{t})=-{\frac {||w_{t}||_{S}^{3}}{Lg_{t}h(w_{t},\gamma _{t})}}}" loading="lazy">.

For each step, if <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle h(w_{t},\gamma _{t})\neq 0}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle h(w_{t},\gamma _{t})\neq 0}</annotation>

</semantics>

</math><img src="./cde3c284d2a4b306d7a5eca3741e12cf716b5a37.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:12.963ex; height:2.843ex;" alt="{\displaystyle h(w_{t},\gamma _{t})\neq 0}" loading="lazy">, then update the direction as

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w_{t+1}=w_{t}-s_{t}\triangledown _{w}f(w_{t},\gamma _{t})}</annotation>

</semantics>

</math><img src="./29ca30755c4a189c3f933cd2149a5399252f8604.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:26.665ex; height:2.843ex;" alt="{\displaystyle w_{t+1}=w_{t}-s_{t}\triangledown _{w}f(w_{t},\gamma _{t})}" loading="lazy">.

Then update the length according to

<msub>

</mrow>

</msub>

<mtext>Bisection</mtext>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma _{t}={\text{Bisection}}(T_{s},f,w_{t})}</annotation>

</semantics>

</math><img src="./aa840ef9457b5a381820c267354c78f2ea4fce05.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:24.415ex; height:2.843ex;" alt="{\displaystyle \gamma _{t}={\text{Bisection}}(T_{s},f,w_{t})}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\text{Bisection()}}}">

<mtext>Bisection()</mtext>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\text{Bisection()}}}</annotation>

</semantics>

</math><img src="./5122dd7bed813f4ea0fae09b75054ef987d0a05b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.089ex; height:2.843ex;" alt="{\displaystyle {\text{Bisection()}}}" loading="lazy"> is the classical <a href="Bisection_algorithm" class="mw-redirect" title="Bisection algorithm">bisection algorithm</a>, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T_{s}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle T_{s}}</annotation>

</semantics>

</math><img src="./942b5ac7697c72b9e3bc1e3b340daf7189fdf61f.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.361ex; height:2.509ex;" alt="{\displaystyle T_{s}}" loading="lazy"> is the total iterations ran in the bisection step.

Denote the total number of iterations as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T_{d}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle T_{d}}</annotation>

</semantics>

</math><img src="./70cadcc969636b990957852f205868f9f7178840.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.45ex; height:2.509ex;" alt="{\displaystyle T_{d}}" loading="lazy">, then the final output of GDNP is

<msub>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<mfrac>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<mrow>

</mrow>

</mrow>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {w}}_{T_{d}}=\gamma _{T_{d}}{\frac {w_{T_{d}}}{||w_{T_{d}}||_{S}}}}</annotation>

</semantics>

</math><img src="./6e005920866895e02728fea73604305f4837df36.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.838ex; width:18.51ex; height:6.009ex;" alt="{\displaystyle {\tilde {w}}_{T_{d}}=\gamma _{T_{d}}{\frac {w_{T_{d}}}{||w_{T_{d}}||_{S}}}}" loading="lazy">.

The GDNP algorithm thus slightly modifies the batch normalization step for the ease of mathematical analysis.

It can be shown that in GDNP, the partial derivative of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{LH}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f_{LH}}</annotation>

</semantics>

</math><img src="./94defc1ff6a3119cb74d8743797f2a9b5055b231.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.95ex; height:2.509ex;" alt="{\displaystyle f_{LH}}" loading="lazy"> against the length component converges to zero at a linear rate, such that

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</msubsup>

<msup>

</mrow>

</msup>

<mfrac>

<mrow>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

<msup>

</mrow>

</msup>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (\partial _{\gamma }f_{LH}(w_{t},a_{t}^{(T_{s})})^{2}\leq {\frac {2^{-T_{s}}\zeta |b_{t}^{(0)}-a_{t}^{(0)}|}{\mu ^{2}}}}</annotation>

</semantics>

</math><img src="./dac4b3d5ec56b983050733ec14eaf3bfb6d6349e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:38.575ex; height:7.009ex;" alt="{\displaystyle (\partial _{\gamma }f_{LH}(w_{t},a_{t}^{(T_{s})})^{2}\leq {\frac {2^{-T_{s}}\zeta |b_{t}^{(0)}-a_{t}^{(0)}|}{\mu ^{2}}}}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle a_{t}^{(0)}}">

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle a_{t}^{(0)}}</annotation>

</semantics>

</math><img src="./655ce27b59661a221a73a4db1c86d535b214cd18.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.563ex; height:3.676ex;" alt="{\displaystyle a_{t}^{(0)}}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b_{t}^{0}}">

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b_{t}^{0}}</annotation>

</semantics>

</math><img src="./80b96c113c5e0ef956ade48ffc16b034b42a1146.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:2.052ex; height:3.176ex;" alt="{\displaystyle b_{t}^{0}}" loading="lazy"> are the two starting points of the bisection algorithm on the left and on the right, correspondingly.

Further, for each iteration, the norm of the gradient of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{LH}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f_{LH}}</annotation>

</semantics>

</math><img src="./94defc1ff6a3119cb74d8743797f2a9b5055b231.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.95ex; height:2.509ex;" alt="{\displaystyle f_{LH}}" loading="lazy"> with respect to <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle w}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle w}</annotation>

</semantics>

</math><img src="./88b1e0c8e1be5ebe69d18a8010676fa42d7961e6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.664ex; height:1.676ex;" alt="{\displaystyle w}" loading="lazy"> converges linearly, such that

</mrow>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</msubsup>

</mrow>

<mfrac>

</mfrac>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</msubsup>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle ||w_{t}||_{S}^{2}||\triangledown f_{LH}(w_{t},g_{t})||_{S^{-1}}^{2}\leq {\bigg (}1-{\frac {\mu }{L}}{\bigg )}^{2t}\Phi ^{2}\gamma _{t}^{2}(\rho (w_{0})-\rho ^{*})}</annotation>

</semantics>

</math><img src="./2e69aa8c10ac98e3226f97873f811f557bc4eda1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:57.323ex; height:6.509ex;" alt="{\displaystyle ||w_{t}||_{S}^{2}||\triangledown f_{LH}(w_{t},g_{t})||_{S^{-1}}^{2}\leq {\bigg (}1-{\frac {\mu }{L}}{\bigg )}^{2t}\Phi ^{2}\gamma _{t}^{2}(\rho (w_{0})-\rho ^{*})}" loading="lazy">.

Combining these two inequalities, a bound could thus be obtained for the gradient with respect to <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tilde {w}}_{T_{d}}}">

<msub>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {w}}_{T_{d}}}</annotation>

</semantics>

</math><img src="./b13d0799f11fb82900a42308223db319bdb3121e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:3.719ex; height:2.843ex;" alt="{\displaystyle {\tilde {w}}_{T_{d}}}" loading="lazy">:

</mrow>

</mrow>

<msub>

<mover>

</mover>

</mrow>

</msub>

<msub>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

<mfrac>

</mfrac>

</mrow>

<msup>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<mfrac>

<mrow>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

<msup>

</mrow>

</msup>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle ||\triangledown _{\tilde {w}}f({\tilde {w}}_{T_{d}})||^{2}\leq {\bigg (}1-{\frac {\mu }{L}}{\bigg )}^{2T_{d}}\Phi ^{2}(\rho (w_{0})-\rho ^{*})+{\frac {2^{-T_{s}}\zeta |b_{t}^{(0)}-a_{t}^{(0)}|}{\mu ^{2}}}}</annotation>

</semantics>

</math><img src="./0489612fd15e3ad03e96d7a1656940429e4e9f05.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:64.961ex; height:7.009ex;" alt="{\displaystyle ||\triangledown _{\tilde {w}}f({\tilde {w}}_{T_{d}})||^{2}\leq {\bigg (}1-{\frac {\mu }{L}}{\bigg )}^{2T_{d}}\Phi ^{2}(\rho (w_{0})-\rho ^{*})+{\frac {2^{-T_{s}}\zeta |b_{t}^{(0)}-a_{t}^{(0)}|}{\mu ^{2}}}}" loading="lazy">, such that the algorithm is guaranteed to converge linearly.

Although the proof stands on the assumption of Gaussian input, it is also shown in experiments that GDNP could accelerate optimization without this constraint.

<div class="mw-heading mw-heading3"><h3 id="Neural_networks">Neural networks</h3></div>

Consider a <a href="Multilayer_perceptron" title="Multilayer perceptron">multilayer perceptron</a> (MLP) with one hidden layer and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle m}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle m}</annotation>

</semantics>

</math><img src="./0a07d98bb302f3856cbabc47b2b9016692e3f7bc.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.04ex; height:1.676ex;" alt="{\displaystyle m}" loading="lazy"> hidden units with mapping from input <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x\in R^{d}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle x\in R^{d}}</annotation>

</semantics>

</math><img src="./23a92052936ccf19989dd3604fc20ffd124440b7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.026ex; height:2.676ex;" alt="{\displaystyle x\in R^{d}}" loading="lazy"> to a scalar output described as

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle F_{x}({\tilde {W}},\Theta )=\sum _{i=1}^{m}\theta _{i}\phi (x^{T}{\tilde {w}}^{(i)})}</annotation>

</semantics>

</math><img src="./94913cebd2254503c351cac4f0a0ab4160ee15d8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:28.141ex; height:6.843ex;" alt="{\displaystyle F_{x}({\tilde {W}},\Theta )=\sum _{i=1}^{m}\theta _{i}\phi (x^{T}{\tilde {w}}^{(i)})}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tilde {w}}^{(i)}}">

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {w}}^{(i)}}</annotation>

</semantics>

</math><img src="./dead04d712f11020730be05360af8f61eccd61fd.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.743ex; height:2.843ex;" alt="{\displaystyle {\tilde {w}}^{(i)}}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \theta _{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \theta _{i}}</annotation>

</semantics>

</math><img src="./302b19204ed378e99ff4575341a67eebdbe5a555.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.89ex; height:2.509ex;" alt="{\displaystyle \theta _{i}}" loading="lazy"> are the input and output weights of unit <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i}</annotation>

</semantics>

</math><img src="./add78d8608ad86e54951b8c8bd6c8d8416533d20.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}" loading="lazy"> correspondingly, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \phi }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \phi }</annotation>

</semantics>

</math><img src="./72b1f30316670aee6270a28334bdf4f5072cdde4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.385ex; height:2.509ex;" alt="{\displaystyle \phi }" loading="lazy"> is the activation function and is assumed to be a <a href="Hyperbolic_function" class="mw-redirect" title="Hyperbolic function">tanh function</a>.

The input and output weights could then be optimized with

<msub>

<mover>

</mover>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle min_{{\tilde {W}},\Theta }(f_{NN}({\tilde {W}},\Theta )=E_{y,x}[l(-yF_{x}({\tilde {W}},\Theta ))])}</annotation>

</semantics>

</math><img src="./99d4f8ac34f55f12699849319e05fa867d3d3c98.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.338ex; width:44.888ex; height:3.676ex;" alt="{\displaystyle min_{{\tilde {W}},\Theta }(f_{NN}({\tilde {W}},\Theta )=E_{y,x}[l(-yF_{x}({\tilde {W}},\Theta ))])}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle l}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle l}</annotation>

</semantics>

</math><img src="./829091f745070b9eb97a80244129025440a1cfac.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.693ex; height:2.176ex;" alt="{\displaystyle l}" loading="lazy"> is a loss function, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tilde {W}}=\{{\tilde {w}}^{(1)},...,{\tilde {w}}^{(m)}\}}">

<mover>

</mover>

</mrow>

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {W}}=\{{\tilde {w}}^{(1)},...,{\tilde {w}}^{(m)}\}}</annotation>

</semantics>

</math><img src="./8366c97137f31a1f915fed43ab8c3b29676d5822.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:21.645ex; height:3.343ex;" alt="{\displaystyle {\tilde {W}}=\{{\tilde {w}}^{(1)},...,{\tilde {w}}^{(m)}\}}" loading="lazy">, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \Theta =\{\theta ^{(1)},...,\theta ^{(m)}\}}">

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Theta =\{\theta ^{(1)},...,\theta ^{(m)}\}}</annotation>

</semantics>

</math><img src="./6a65c783ef8a8eb0acb273071c38d96ad298f055.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.87ex; height:3.343ex;" alt="{\displaystyle \Theta =\{\theta ^{(1)},...,\theta ^{(m)}\}}" loading="lazy">.

Consider fixed <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \Theta }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Theta }</annotation>

</semantics>

</math><img src="./bc927b19f46d005b4720db7a0f96cd5b6f1a0d9b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.808ex; height:2.176ex;" alt="{\displaystyle \Theta }" loading="lazy"> and optimizing only <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\tilde {W}}}">

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\tilde {W}}}</annotation>

</semantics>

</math><img src="./0dae50f239be9486a2273ec98c497e983610fab1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.435ex; height:2.676ex;" alt="{\displaystyle {\tilde {W}}}" loading="lazy">, it can be shown that the critical points of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{NN}({\tilde {W}})}">

<msub>

</mrow>

</msub>

<mover>

</mover>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f_{NN}({\tilde {W}})}</annotation>

</semantics>

</math><img src="./69c185c5fce2d4eadfd6477b3350d07ce1cff644.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.534ex; height:3.176ex;" alt="{\displaystyle f_{NN}({\tilde {W}})}" loading="lazy"> of a particular hidden unit <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i}</annotation>

</semantics>

</math><img src="./add78d8608ad86e54951b8c8bd6c8d8416533d20.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}" loading="lazy">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {w}}^{(i)}}">

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {w}}^{(i)}}</annotation>

</semantics>

</math><img src="./60d1b4ae6b98309cc9035090a9fb3ddd61785b22.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.743ex; height:2.843ex;" alt="{\displaystyle {\hat {w}}^{(i)}}" loading="lazy">, all align along one line depending on incoming information into the hidden layer, such that

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {w}}^{(i)}={\hat {c}}^{(i)}S^{-1}u}</annotation>

</semantics>

</math><img src="./c9d8954d7eaad91a6a8c0df63d3e0a36d3d5d5b6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:15.396ex; height:2.843ex;" alt="{\displaystyle {\hat {w}}^{(i)}={\hat {c}}^{(i)}S^{-1}u}" loading="lazy">, where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\hat {c}}^{(i)}\in R}">

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\hat {c}}^{(i)}\in R}</annotation>

</semantics>

</math><img src="./3f6e6f405115516bd625706391cec3f299c115c2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.975ex; height:2.843ex;" alt="{\displaystyle {\hat {c}}^{(i)}\in R}" loading="lazy"> is a scalar, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i=1,...,m}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i=1,...,m}</annotation>

</semantics>

</math><img src="./f883868ad60f110c23795233999f65424ce2edc8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:12.273ex; height:2.509ex;" alt="{\displaystyle i=1,...,m}" loading="lazy">.

This result could be proved by setting the gradient of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f_{NN}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f_{NN}}</annotation>

</semantics>

</math><img src="./08a9d666847a80de2fc08867864b3df53f83b651.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.29ex; height:2.509ex;" alt="{\displaystyle f_{NN}}" loading="lazy"> to zero and solving the system of equations.

Apply the GDNP algorithm to this optimization problem by alternating optimization over the different hidden units. Specifically, for each hidden unit, run GDNP to find the optimal <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W}</annotation>

</semantics>

</math><img src="./54a9c4c547f4d6111f81946cad242b18298d70b7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.435ex; height:2.176ex;" alt="{\displaystyle W}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \gamma }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \gamma }</annotation>

</semantics>

</math><img src="./a223c880b0ce3da8f64ee33c4f0010beee400b1a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:1.262ex; height:2.176ex;" alt="{\displaystyle \gamma }" loading="lazy">. With the same choice of stopping criterion and stepsize, it follows that

</mrow>

</mrow>

<msub>

<msup>

<mover>

</mover>

</mrow>

</mrow>

</msup>

</mrow>

</msub>

<mover>

</mover>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</msubsup>

</mrow>

<mfrac>

</mfrac>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</msup>

<mfrac>

<mrow>

<msup>

</mrow>

</mrow>

</msubsup>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

<msup>

</mrow>

</msup>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle ||\triangledown _{{\tilde {w}}^{(i)}}f({\tilde {w}}_{t}^{(i)})||_{S^{-1}}^{2}\leq {\bigg (}1-{\frac {\mu }{L}}{\bigg )}^{2t}C(\rho (w_{0})-\rho ^{*})+{\frac {2^{-T_{s}^{(i)}}\zeta |b_{t}^{(0)}-a_{t}^{(0)}|}{\mu ^{2}}}}</annotation>

</semantics>

</math><img src="./433aac5aef62e7c3e0b249a06d84ef9f14d4829a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:67.708ex; height:7.509ex;" alt="{\displaystyle ||\triangledown _{{\tilde {w}}^{(i)}}f({\tilde {w}}_{t}^{(i)})||_{S^{-1}}^{2}\leq {\bigg (}1-{\frac {\mu }{L}}{\bigg )}^{2t}C(\rho (w_{0})-\rho ^{*})+{\frac {2^{-T_{s}^{(i)}}\zeta |b_{t}^{(0)}-a_{t}^{(0)}|}{\mu ^{2}}}}" loading="lazy">.

Since the parameters of each hidden unit converge linearly, the whole optimization problem has a linear rate of convergence.<a href="#cite_note-:2-8">[8]</a>

<div class="mw-heading mw-heading2"><h2 id="References">References</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}

/* end https://en.wikipedia.org/ */

</style><div class="reflist">

/* start https://en.wikipedia.org/ */

.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("./mw/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("./mw/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("./mw/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("./mw/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}

/* end https://en.wikipedia.org/ */

</style><cite id="CITEREFIoffeSzegedy2015" class="citation arxiv cs1">Ioffe, Sergey; Szegedy, Christian (2015). "Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1502.03167">1502.03167</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite>

</li>

<li id="cite_note-:13-2"><a href="#cite_ref-:13_2-0">^</a> <cite id="CITEREFSanturkarTsiprasIlyasMadry2018" class="citation arxiv cs1">Santurkar, Shibani; Tsipras, Dimitris; Ilyas, Andrew; Madry, Aleksander (29 May 2018). "How Does Batch Normalization Help Optimization?". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1805.11604">1805.11604</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite>

</li>

<li id="cite_note-:73-3"><a href="#cite_ref-:73_3-0">^</a> <cite id="CITEREFYangPenningtonRaoSohl-Dickstein2019" class="citation arxiv cs1">Yang, Greg; Pennington, Jeffrey; Rao, Vinay; Sohl-Dickstein, Jascha; Schoenholz, Samuel S. (2019). "A Mean Field Theory of Batch Normalization". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1902.08129">1902.08129</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite>

</li>

<li id="cite_note-:23-4"><a href="#cite_ref-:23_4-0">^</a> <cite id="CITEREFKohlerDaneshmandLucchiZhou2018" class="citation arxiv cs1">Kohler, Jonas; Daneshmand, Hadi; Lucchi, Aurelien; Zhou, Ming; Neymeyr, Klaus; Hofmann, Thomas (27 May 2018). "Exponential convergence rates for Batch Normalization: The power of length-direction decoupling in non-convex optimization". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1805.10694">1805.10694</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite>

</li>

<li id="cite_note-:1-5">^ <a href="#cite_ref-:1_5-0">a</a> <a href="#cite_ref-:1_5-1">b</a> <a href="#cite_ref-:1_5-2">c</a> <cite id="CITEREFSanturkarTsiprasIlyasMadry2018" class="citation arxiv cs1">Santurkar, Shibani; Tsipras, Dimitris; Ilyas, Andrew; Madry, Aleksander (29 May 2018). "How Does Batch Normalization Help Optimization?". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1805.11604">1805.11604</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite>

</li>

<li id="cite_note-VGG-6">^ <a href="#cite_ref-VGG_6-0">a</a> <a href="#cite_ref-VGG_6-1">b</a> <cite id="CITEREFSimonyanAndrew2014" class="citation arxiv cs1">Simonyan, Karen; Andrew, Zisserman (2014). "Very Deep Convolution Networks for Large Scale Image Recognition". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1409.1556">1409.1556</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CV">cs.CV</a>].</cite>

</li>

<li id="cite_note-7"><a href="#cite_ref-7">^</a> Ba, J., Kiros, J.R., & Hinton, G.E. (2016). Layer Normalization. ArXiv, abs/1607.06450.

</li>

<li id="cite_note-:2-8">^ <a href="#cite_ref-:2_8-0">a</a> <a href="#cite_ref-:2_8-1">b</a> <cite id="CITEREFKohlerDaneshmandLucchiZhou2018" class="citation arxiv cs1">Kohler, Jonas; Daneshmand, Hadi; Lucchi, Aurelien; Zhou, Ming; Neymeyr, Klaus; Hofmann, Thomas (27 May 2018). "Exponential convergence rates for Batch Normalization: The power of length-direction decoupling in non-convex optimization". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1805.10694">1805.10694</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/stat.ML">stat.ML</a>].</cite>

</li>

<li id="cite_note-:7-9"><a href="#cite_ref-:7_9-0">^</a> <cite id="CITEREFYangPenningtonRaoSohl-Dickstein2019" class="citation arxiv cs1">Yang, Greg; Pennington, Jeffrey; Rao, Vinay; Sohl-Dickstein, Jascha; Schoenholz, Samuel S. (2019). "A Mean Field Theory of Batch Normalization". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1902.08129">1902.08129</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite>

</li>

<li id="cite_note-10"><a href="#cite_ref-10">^</a> <cite id="CITEREFKnyazev2003" class="citation journal cs1">Knyazev, Neymeyr (2003). <a rel="nofollow" class="external text" href="https://doi.org/10.1016%2FS0024-3795%2801%2900461-X">"A geometric theory for preconditioned inverse iteration III: A short and sharp convergence estimate for generalized eigenvalue problems"</a>. Linear Algebra and Its Applications. 358 (1–3): 95–114. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2FS0024-3795%2801%2900461-X">10.1016/S0024-3795(01)00461-X</a>.</cite>

</li>

</ol></div></div>

<div class="mw-heading mw-heading2"><h2 id="Further_reading">Further reading</h2></div>

<ul><li>Ioffe, Sergey; Szegedy, Christian (2015). "Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift", ICML'15: Proceedings of the 32nd International Conference on International Conference on Machine Learning - Volume 37, July 2015 Pages 448–456</li>

<li><cite id="CITEREFSimonyanZisserman2014" class="citation arxiv cs1">Simonyan, Karen; Zisserman, Andrew (2014). "Very Deep Convolutional Networks for Large-Scale Image Recognition". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1409.1556">1409.1556</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CV">cs.CV</a>].</cite></li></ul>

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></div><div role="navigation" class="navbox" aria-labelledby="Artificial_intelligence_(AI)426" style="padding:3px"><table class="nowraplinks hlist mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Artificial_intelligence_(AI)426" style="font-size:114%;margin:0 4em"><a href="Artificial_intelligence" title="Artificial intelligence">Artificial intelligence</a> (AI)</div></th></tr><tr><td class="navbox-abovebelow" colspan="2"><div>

<ul><li><a href="History_of_artificial_intelligence" title="History of artificial intelligence">History</a>

<ul><li><a href="Timeline_of_artificial_intelligence" title="Timeline of artificial intelligence">timeline</a></li></ul></li>

<li><a href="List_of_artificial_intelligence_companies" title="List of artificial intelligence companies">Companies</a></li>

<li><a href="List_of_artificial_intelligence_projects" title="List of artificial intelligence projects">Projects</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Concepts</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Parameter" title="Parameter">Parameter</a>

<ul><li><a href="Hyperparameter_(machine_learning)" title="Hyperparameter (machine learning)">Hyperparameter</a></li></ul></li>

<li><a href="Loss_functions_for_classification" title="Loss functions for classification">Loss functions</a></li>

<li><a href="Regression_analysis" title="Regression analysis">Regression</a>

<ul><li><a href="Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li>

<li><a href="Double_descent" title="Double descent">Double descent</a></li>

<li><a href="Overfitting" title="Overfitting">Overfitting</a></li></ul></li>

<li><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></li>

<li><a href="Gradient_descent" title="Gradient descent">Gradient descent</a>

<li><a href="Quasi-Newton_method" title="Quasi-Newton method">Quasi-Newton method</a></li>

<li><a href="Conjugate_gradient_method" title="Conjugate gradient method">Conjugate gradient method</a></li></ul></li>

<li><a href="Backpropagation" title="Backpropagation">Backpropagation</a></li>

<li><a href="Attention_(machine_learning)" title="Attention (machine learning)">Attention</a></li>

<li><a href="Convolution" title="Convolution">Convolution</a></li>

<li><a href="Normalization_(machine_learning)" title="Normalization (machine learning)">Normalization</a>

<li><a href="Activation_function" title="Activation function">Activation</a>

<ul><li><a href="Softmax_function" title="Softmax function">Softmax</a></li>

<li><a href="Sigmoid_function" title="Sigmoid function">Sigmoid</a></li>

<li><a href="Rectifier_(neural_networks)" title="Rectifier (neural networks)">Rectifier</a></li></ul></li>

<li><a href="Gating_mechanism" title="Gating mechanism">Gating</a></li>

<li><a href="Weight_initialization" title="Weight initialization">Weight initialization</a></li>

<li><a href="Regularization_(mathematics)" title="Regularization (mathematics)">Regularization</a></li>

<li><a href="Training%2C_validation%2C_and_test_data_sets" title="Training, validation, and test data sets">Datasets</a>

<ul><li><a href="Data_augmentation" title="Data augmentation">Augmentation</a></li></ul></li>

<li><a href="Prompt_engineering" title="Prompt engineering">Prompt engineering</a></li>

<li><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a>

<ul><li><a href="Q-learning" title="Q-learning">Q-learning</a></li>

<li><a href="State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li>

<li><a href="Imitation_learning" title="Imitation learning">Imitation</a></li>

<li><a href="Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li></ul></li>

<li><a href="Diffusion_process" title="Diffusion process">Diffusion</a></li>

<li><a href="Latent_diffusion_model" title="Latent diffusion model">Latent diffusion model</a></li>

<li><a href="Autoregressive_model" title="Autoregressive model">Autoregression</a></li>

<li><a href="Adversarial_machine_learning" title="Adversarial machine learning">Adversary</a></li>

<li><a href="Uncanny_valley" title="Uncanny valley">Uncanny valley</a></li>

<li><a href="Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li>

<li><a href="Reflection_(artificial_intelligence)" class="mw-redirect" title="Reflection (artificial intelligence)">Reflection</a></li>

<li><a href="Recursive_self-improvement" title="Recursive self-improvement">Recursive self-improvement</a></li>

<li><a href="Hallucination_(artificial_intelligence)" title="Hallucination (artificial intelligence)">Hallucination</a></li>

<li><a href="Word_embedding" title="Word embedding">Word embedding</a></li>

<li><a href="Vibe_coding" title="Vibe coding">Vibe coding</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Applications</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Machine_learning" title="Machine learning">Machine learning</a>

<ul><li><a href="Prompt_engineering#In-context_learning" title="Prompt engineering">In-context learning</a></li></ul></li>

<li><a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">Artificial neural network</a>

<ul><li><a href="Deep_learning" title="Deep learning">Deep learning</a></li></ul></li>

<li><a href="Language_model" title="Language model">Language model</a>

<ul><li><a href="Large_language_model" title="Large language model">Large language model</a></li>

<li><a href="Reasoning_language_model" title="Reasoning language model">Reasoning language model</a></li>

<li><a href="Model_Context_Protocol" title="Model Context Protocol">Model Context Protocol</a></li>

<li><a href="Intelligent_agent" title="Intelligent agent">Intelligent agent</a></li>

<li><a href="Artificial_human_companion" title="Artificial human companion">Artificial human companion</a></li>

<li><a href="Humanity's_Last_Exam" title="Humanity's Last Exam">Humanity's Last Exam</a></li>

<li><a href="Artificial_general_intelligence" title="Artificial general intelligence">Artificial general intelligence (AGI)</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Implementations</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%">Audio–visual</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="AlexNet" title="AlexNet">AlexNet</a></li>

<li><a href="WaveNet" title="WaveNet">WaveNet</a></li>

<li><a href="Human_image_synthesis" title="Human image synthesis">Human image synthesis</a></li>

<li><a href="Computer_vision" title="Computer vision">Computer vision</a></li>

<li><a href="Deep_learning_speech_synthesis" title="Deep learning speech synthesis">Speech synthesis</a>

<li><a href="ElevenLabs" title="ElevenLabs">ElevenLabs</a></li></ul></li>

<li><a href="Speech_recognition" title="Speech recognition">Speech recognition</a>

<ul><li><a href="Whisper_(speech_recognition_system)" title="Whisper (speech recognition system)">Whisper</a></li></ul></li>

<li><a href="Facial_recognition_system" title="Facial recognition system">Facial recognition</a></li>

<li><a href="AlphaFold" title="AlphaFold">AlphaFold</a></li>

<li><a href="Text-to-image_model" title="Text-to-image model">Text-to-image models</a>

<ul><li><a href="Aurora_(text-to-image_model)" class="mw-redirect" title="Aurora (text-to-image model)">Aurora</a></li>

<li><a href="Adobe_Firefly" title="Adobe Firefly">Firefly</a></li>

<li><a href="Ideogram_(text-to-image_model)" title="Ideogram (text-to-image model)">Ideogram</a></li>

<li><a href="Imagen_(text-to-image_model)" title="Imagen (text-to-image model)">Imagen</a></li>

<li><a href="Midjourney" title="Midjourney">Midjourney</a></li>

<li><a href="Recraft" title="Recraft">Recraft</a></li>

<li><a href="Stable_Diffusion" title="Stable Diffusion">Stable Diffusion</a></li></ul></li>

<li><a href="Text-to-video_model" title="Text-to-video model">Text-to-video models</a>

<ul><li><a href="Dream_Machine_(text-to-video_model)" title="Dream Machine (text-to-video model)">Dream Machine</a></li>

<li><a href="Runway_(company)#Services_and_technologies" title="Runway (company)">Runway Gen</a></li>

<li><a href="MiniMax_(company)#Hailuo_AI" title="MiniMax (company)">Hailuo AI</a></li>

<li><a href="Kling_(text-to-video_model)" class="mw-redirect" title="Kling (text-to-video model)">Kling</a></li>

<li><a href="Music_and_artificial_intelligence" title="Music and artificial intelligence">Music generation</a>

<ul><li><a href="Riffusion" title="Riffusion">Riffusion</a></li>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Text</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<li><a href="GloVe" title="GloVe">GloVe</a></li>

<li><a href="Llama_(language_model)" title="Llama (language model)">Llama</a></li>

<li><a href="Chinchilla_(language_model)" title="Chinchilla (language model)">Chinchilla AI</a></li>

<li><a href="ChatGPT" title="ChatGPT">ChatGPT</a></li>

<li><a href="Claude_(language_model)" title="Claude (language model)">Claude</a></li>

<li><a href="Gemini_(language_model)" title="Gemini (language model)">Gemini</a>

<ul><li><a href="Gemini_(chatbot)" title="Gemini (chatbot)">chatbot</a></li></ul></li>

<li><a href="LaMDA" title="LaMDA">LaMDA</a></li>

<li><a href="BLOOM_(language_model)" title="BLOOM (language model)">BLOOM</a></li>

<li><a href="Project_Debater" title="Project Debater">Project Debater</a></li>

<li><a href="IBM_Watson" title="IBM Watson">IBM Watson</a></li>

<li><a href="IBM_Watsonx" title="IBM Watsonx">IBM Watsonx</a></li>

<li><a href="IBM_Granite" title="IBM Granite">Granite</a></li>

<li><a href="Huawei_PanGu" title="Huawei PanGu">PanGu-Σ</a></li>

<li><a href="DeepSeek_(chatbot)" title="DeepSeek (chatbot)">DeepSeek</a></li>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Decisional</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="AlphaGo" title="AlphaGo">AlphaGo</a></li>

<li><a href="AlphaZero" title="AlphaZero">AlphaZero</a></li>

<li><a href="OpenAI_Five" title="OpenAI Five">OpenAI Five</a></li>

<li><a href="Self-driving_car" title="Self-driving car">Self-driving car</a></li>

<li><a href="MuZero" title="MuZero">MuZero</a></li>

<li><a href="Action_selection" title="Action selection">Action selection</a>

<ul><li><a href="AutoGPT" title="AutoGPT">AutoGPT</a></li></ul></li>

<li><a href="Robot_control" title="Robot control">Robot control</a></li></ul>

</div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">People</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Alan_Turing" title="Alan Turing">Alan Turing</a></li>

<li><a href="Warren_Sturgis_McCulloch" title="Warren Sturgis McCulloch">Warren Sturgis McCulloch</a></li>

<li><a href="Walter_Pitts" title="Walter Pitts">Walter Pitts</a></li>

<li><a href="John_von_Neumann" title="John von Neumann">John von Neumann</a></li>

<li><a href="Claude_Shannon" title="Claude Shannon">Claude Shannon</a></li>

<li><a href="Shun'ichi_Amari" title="Shun'ichi Amari">Shun'ichi Amari</a></li>

<li><a href="Kunihiko_Fukushima" title="Kunihiko Fukushima">Kunihiko Fukushima</a></li>

<li><a href="Takeo_Kanade" title="Takeo Kanade">Takeo Kanade</a></li>

<li><a href="Marvin_Minsky" title="Marvin Minsky">Marvin Minsky</a></li>

<li><a href="John_McCarthy_(computer_scientist)" title="John McCarthy (computer scientist)">John McCarthy</a></li>

<li><a href="Nathaniel_Rochester_(computer_scientist)" title="Nathaniel Rochester (computer scientist)">Nathaniel Rochester</a></li>

<li><a href="Allen_Newell" title="Allen Newell">Allen Newell</a></li>

<li><a href="Cliff_Shaw" title="Cliff Shaw">Cliff Shaw</a></li>

<li><a href="Herbert_A._Simon" title="Herbert A. Simon">Herbert A. Simon</a></li>

<li><a href="Oliver_Selfridge" title="Oliver Selfridge">Oliver Selfridge</a></li>

<li><a href="Frank_Rosenblatt" title="Frank Rosenblatt">Frank Rosenblatt</a></li>

<li><a href="Bernard_Widrow" title="Bernard Widrow">Bernard Widrow</a></li>

<li><a href="Joseph_Weizenbaum" title="Joseph Weizenbaum">Joseph Weizenbaum</a></li>

<li><a href="Seymour_Papert" title="Seymour Papert">Seymour Papert</a></li>

<li><a href="Seppo_Linnainmaa" title="Seppo Linnainmaa">Seppo Linnainmaa</a></li>

<li><a href="Paul_Werbos" title="Paul Werbos">Paul Werbos</a></li>

<li><a href="Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a></li>

<li><a href="John_Hopfield" title="John Hopfield">John Hopfield</a></li>

<li><a href="J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Jürgen Schmidhuber</a></li>

<li><a href="Yann_LeCun" title="Yann LeCun">Yann LeCun</a></li>

<li><a href="Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a></li>

<li><a href="Lotfi_A._Zadeh" title="Lotfi A. Zadeh">Lotfi A. Zadeh</a></li>

<li><a href="Stephen_Grossberg" title="Stephen Grossberg">Stephen Grossberg</a></li>

<li><a href="Alex_Graves_(computer_scientist)" title="Alex Graves (computer scientist)">Alex Graves</a></li>

<li><a href="James_Goodnight" title="James Goodnight">James Goodnight</a></li>

<li><a href="Andrew_Ng" title="Andrew Ng">Andrew Ng</a></li>

<li><a href="Ilya_Sutskever" title="Ilya Sutskever">Ilya Sutskever</a></li>

<li><a href="Alex_Krizhevsky" title="Alex Krizhevsky">Alex Krizhevsky</a></li>

<li><a href="Ian_Goodfellow" title="Ian Goodfellow">Ian Goodfellow</a></li>

<li><a href="Demis_Hassabis" title="Demis Hassabis">Demis Hassabis</a></li>

<li><a href="David_Silver_(computer_scientist)" title="David Silver (computer scientist)">David Silver</a></li>

<li><a href="Andrej_Karpathy" title="Andrej Karpathy">Andrej Karpathy</a></li>

<li><a href="Ashish_Vaswani" title="Ashish Vaswani">Ashish Vaswani</a></li>

<li><a href="Noam_Shazeer" title="Noam Shazeer">Noam Shazeer</a></li>

<li><a href="Aidan_Gomez" title="Aidan Gomez">Aidan Gomez</a></li>

<li><a href="Mustafa_Suleyman" title="Mustafa Suleyman">Mustafa Suleyman</a></li>

<li><a href="Fran%C3%A7ois_Chollet" title="François Chollet">François Chollet</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Architectures</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Neural_Turing_machine" title="Neural Turing machine">Neural Turing machine</a></li>

<li><a href="Differentiable_neural_computer" title="Differentiable neural computer">Differentiable neural computer</a></li>

<li><a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>

<ul><li><a href="Vision_transformer" title="Vision transformer">Vision transformer (ViT)</a></li></ul></li>

<li><a href="Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network (RNN)</a></li>

<li><a href="Long_short-term_memory" title="Long short-term memory">Long short-term memory (LSTM)</a></li>

<li><a href="Gated_recurrent_unit" title="Gated recurrent unit">Gated recurrent unit (GRU)</a></li>

<li><a href="Echo_state_network" title="Echo state network">Echo state network</a></li>

<li><a href="Multilayer_perceptron" title="Multilayer perceptron">Multilayer perceptron (MLP)</a></li>

<li><a href="Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network (CNN)</a></li>

<li><a href="Residual_neural_network" title="Residual neural network">Residual neural network (RNN)</a></li>

<li><a href="Highway_network" title="Highway network">Highway network</a></li>

<li><a href="Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li>

<li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Variational_autoencoder" title="Variational autoencoder">Variational autoencoder (VAE)</a></li>

<li><a href="Generative_adversarial_network" title="Generative adversarial network">Generative adversarial network (GAN)</a></li>

<li><a href="Graph_neural_network" title="Graph neural network">Graph neural network (GNN)</a></li></ul>

</div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div>

<ul><li> Category</li></ul>

</div></td></tr></tbody></table></div></div><div><div class="zim-footer">

This article is issued from <a class="external text" title="Last edited on 2025-05-16" href="https://en.wikipedia.org/wiki/?title=Batch_normalization&oldid=1290651114">Wikipedia</a>. The text is available under <a class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-Share Alike 4.0</a> unless otherwise noted. Additional terms may apply for the media files.

</div>

</div></div>

</div>

</main>

</div>

</body></html>